From da2ec5ee995d8c1f60d46a4319c38a81fe4b89a9 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Bala Krishnan Date: Wed, 19 Aug 2026 11:52:15 +0000 Subject: [PATCH 01/49] Add Datadog credential issuance Co-authored-by: c1-squire-dev[bot] --- pkg/client/client.go | 36 ++++++++++++++++++++++++++++++++++++ pkg/connector/api_token.go | 12 ++++++++++++ pkg/connector/connector.go | 6 +++++- pkg/connector/users.go | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 88 insertions(+), 1 deletion(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index a741554d..a77bc779 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -179,6 +179,42 @@ func (w *DatadogClient) ListAPIKeys(ctx context.Context, params *datadogV2.ListA return &resp, nil } +type IssuedAPIKey struct { + ID string + Secret string +} + +func (w *DatadogClient) CreateAPIKey(ctx context.Context, name string) (*IssuedAPIKey, error) { + ctx = w.withAuthContext(ctx) + api := datadogV2.NewKeyManagementApi(w.officialClient) + attrs := *datadogV2.NewAPIKeyCreateAttributes(name) + data := *datadogV2.NewAPIKeyCreateData(attrs, datadogV2.APIKEYSTYPE_API_KEYS) + response, httpRes, err := api.CreateAPIKey(ctx, *datadogV2.NewAPIKeyCreateRequest(data)) + if httpRes != nil { + defer httpRes.Body.Close() + } + if err != nil { + return nil, wrapOfficialClientError("create API key", httpRes, err) + } + if response.Data.Id == nil || response.Data.Attributes == nil || response.Data.Attributes.Key == nil || *response.Data.Attributes.Key == "" { + return nil, fmt.Errorf("create API key response omitted id or key") + } + return &IssuedAPIKey{ID: *response.Data.Id, Secret: *response.Data.Attributes.Key}, nil +} + +func (w *DatadogClient) DeleteAPIKey(ctx context.Context, id string) error { + ctx = w.withAuthContext(ctx) + api := datadogV2.NewKeyManagementApi(w.officialClient) + httpRes, err := api.DeleteAPIKey(ctx, id) + if httpRes != nil { + defer httpRes.Body.Close() + } + if err != nil { + return wrapOfficialClientError("delete API key", httpRes, err) + } + return nil +} + // Wrapper methods that handle HTTP response body closing automatically // ListRoleUsers lists users for a specific role and automatically handles HTTP response body closing. diff --git a/pkg/connector/api_token.go b/pkg/connector/api_token.go index 82576650..7f55c8c7 100644 --- a/pkg/connector/api_token.go +++ b/pkg/connector/api_token.go @@ -8,6 +8,7 @@ import ( "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" "github.com/conductorone/baton-datadog/pkg/client" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" "github.com/conductorone/baton-sdk/pkg/types/resource" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" @@ -20,6 +21,17 @@ type apiTokenBuilder struct { } var _ connectorbuilder.ResourceSyncerV2 = &apiTokenBuilder{} +var _ connectorbuilder.ResourceDeleterV2Limited = &apiTokenBuilder{} + +func (o *apiTokenBuilder) Delete(ctx context.Context, resourceID *v2.ResourceId, _ *v2.ResourceId) (annotations.Annotations, error) { + if resourceID == nil || resourceID.GetResource() == "" { + return nil, fmt.Errorf("baton-datadog: API key id is required") + } + if err := o.wrapper.DeleteAPIKey(ctx, resourceID.GetResource()); err != nil { + return nil, fmt.Errorf("baton-datadog: delete API key: %w", err) + } + return nil, nil +} func (o *apiTokenBuilder) Entitlements(_ context.Context, _ *v2.Resource, _ resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) { // API Token secrets do not have entitlements diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index 26bf339c..aef6dee6 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -34,8 +34,12 @@ type Datadog struct { // ResourceSyncers returns a ResourceSyncer for each resource type that should be synced from the upstream service. func (d *Datadog) ResourceSyncers(ctx context.Context) []connectorbuilder.ResourceSyncerV2 { + userSyncer := connectorbuilder.ResourceSyncerV2(newUserBuilder(d.wrapper)) + if d.SyncSecrets { + userSyncer = newCredentialUserBuilder(d.wrapper) + } resourceSyncers := []connectorbuilder.ResourceSyncerV2{ - newUserBuilder(d.wrapper), + userSyncer, newTeamBuilder(d.wrapper), newRoleBuilder(d.wrapper), } diff --git a/pkg/connector/users.go b/pkg/connector/users.go index b676ff7e..273edb72 100644 --- a/pkg/connector/users.go +++ b/pkg/connector/users.go @@ -19,6 +19,41 @@ type userBuilder struct { wrapper *client.DatadogClient } +type credentialUserBuilder struct{ *userBuilder } + +func newCredentialUserBuilder(wrapper *client.DatadogClient) *credentialUserBuilder { + return &credentialUserBuilder{userBuilder: newUserBuilder(wrapper)} +} + +func (u *credentialUserBuilder) IssueCapabilityDetails(context.Context) (*v2.CredentialDetailsCredentialIssue, annotations.Annotations, error) { + return v2.CredentialDetailsCredentialIssue_builder{ + Options: []*v2.CredentialIssueOptionDescriptor{v2.CredentialIssueOptionDescriptor_builder{ + Option: v2.CapabilityDetailCredentialOption_CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY, + ResourceMode: v2.CredentialResourceMode_CREDENTIAL_RESOURCE_MODE_DISCOVERABLE, + SecretResourceTypeId: apiTokenResourceType.Id, + }.Build()}, + PreferredOption: v2.CapabilityDetailCredentialOption_CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY, + }.Build(), nil, nil +} + +func (u *credentialUserBuilder) Issue(ctx context.Context, input *connectorbuilder.CredentialIssueInput) (*connectorbuilder.CredentialIssueOutput, error) { + if input == nil || input.IdentityID == nil || input.IdentityID.GetResourceType() != userResourceType.Id { + return nil, fmt.Errorf("baton-datadog: a Datadog user identity is required") + } + name := "c1-" + input.RequestID + key, err := u.wrapper.CreateAPIKey(ctx, name) + if err != nil { + return nil, fmt.Errorf("baton-datadog: create API key: %w", err) + } + secret, err := rs.NewSecretResource(name, apiTokenResourceType, key.ID, + []rs.SecretTraitOption{rs.WithSecretCreatedByID(input.IdentityID), rs.WithSecretIdentityID(input.IdentityID), rs.WithSecretType(v2.SecretTrait_CREDENTIAL_TYPE_STATIC_SECRET), rs.WithSecretDetail("datadog.api_key")}, + ) + if err != nil { + return nil, err + } + return &connectorbuilder.CredentialIssueOutput{Secret: secret, PlaintextData: []*v2.PlaintextData{v2.PlaintextData_builder{Name: "api_key", Bytes: []byte(key.Secret)}.Build()}, ResourceMode: v2.CredentialResourceMode_CREDENTIAL_RESOURCE_MODE_DISCOVERABLE}, nil +} + var _ connectorbuilder.ResourceSyncerV2 = &userBuilder{} var _ connectorbuilder.AccountManagerV2 = &userBuilder{} var _ connectorbuilder.ResourceActionProvider = &userBuilder{} From c89c75b38ff4d19d43707c75e0c0ed89093936dc Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Bala Krishnan Date: Thu, 20 Aug 2026 05:00:53 +0000 Subject: [PATCH 02/49] Update Datadog credential metadata Co-authored-by: c1-squire-dev[bot] --- baton_capabilities.json | 20 +++++++++++++++++--- docs/connector.mdx | 18 ++++++++++-------- pkg/connector/users.go | 18 ++++++++++++++---- 3 files changed, 41 insertions(+), 15 deletions(-) diff --git a/baton_capabilities.json b/baton_capabilities.json index e95d347f..21f23acf 100644 --- a/baton_capabilities.json +++ b/baton_capabilities.json @@ -23,7 +23,8 @@ ] }, "capabilities": [ - "CAPABILITY_SYNC" + "CAPABILITY_SYNC", + "CAPABILITY_RESOURCE_DELETE" ], "permissions": { "permissions": [ @@ -148,7 +149,8 @@ }, "capabilities": [ "CAPABILITY_SYNC", - "CAPABILITY_ACCOUNT_PROVISIONING" + "CAPABILITY_ACCOUNT_PROVISIONING", + "CAPABILITY_CREDENTIAL_ISSUE" ], "permissions": { "permissions": [ @@ -159,6 +161,16 @@ "permission": "user_access_manage" } ] + }, + "credentialIssue": { + "options": [ + { + "option": "CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY", + "resourceMode": "CREDENTIAL_RESOURCE_MODE_DISCOVERABLE", + "secretResourceTypeId": "api-key" + } + ], + "preferredOption": "CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY" } } ], @@ -166,7 +178,9 @@ "CAPABILITY_PROVISION", "CAPABILITY_SYNC", "CAPABILITY_ACCOUNT_PROVISIONING", - "CAPABILITY_ACTIONS" + "CAPABILITY_RESOURCE_DELETE", + "CAPABILITY_ACTIONS", + "CAPABILITY_CREDENTIAL_ISSUE" ], "credentialDetails": { "capabilityAccountProvisioning": { diff --git a/docs/connector.mdx b/docs/connector.mdx index a744663e..5ac56d01 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -12,16 +12,18 @@ sidebarTitle: "Datadog" ## Capabilities -| Resource | Sync | Provision | -| :--- | :--- | :--- | -| Accounts | | | -| Roles | | | -| Teams | | | -| Schedules | * | | -| Secrets - API keys | | | +| Resource | Sync | Provision | Issue | Revoke | +| :--- | :--- | :--- | :--- | :--- | +| Accounts | | | | | +| Roles | | | | | +| Teams | | | | | +| Schedules | * | | | | +| Secrets - API keys | | | | | [This connector can sync secrets](/product/admin/inventory) and display them on the **Inventory** page. +API keys can be issued and revoked through C1. Datadog does not support an expiration date when creating an API key. + *Schedules are not synced by default, but you can opt into syncing them when configuring the connector. ### Connector actions @@ -281,4 +283,4 @@ spec: **Done.** Your Datadog connector is now pulling access data into C1. - \ No newline at end of file + diff --git a/pkg/connector/users.go b/pkg/connector/users.go index 273edb72..cb9759ce 100644 --- a/pkg/connector/users.go +++ b/pkg/connector/users.go @@ -45,13 +45,23 @@ func (u *credentialUserBuilder) Issue(ctx context.Context, input *connectorbuild if err != nil { return nil, fmt.Errorf("baton-datadog: create API key: %w", err) } - secret, err := rs.NewSecretResource(name, apiTokenResourceType, key.ID, - []rs.SecretTraitOption{rs.WithSecretCreatedByID(input.IdentityID), rs.WithSecretIdentityID(input.IdentityID), rs.WithSecretType(v2.SecretTrait_CREDENTIAL_TYPE_STATIC_SECRET), rs.WithSecretDetail("datadog.api_key")}, - ) + secretTraitOptions := []rs.SecretTraitOption{ + rs.WithSecretCreatedByID(input.IdentityID), + rs.WithSecretIdentityID(input.IdentityID), + rs.WithSecretType(v2.SecretTrait_CREDENTIAL_TYPE_STATIC_SECRET), + rs.WithSecretDetail("datadog.api_key"), + } + secret, err := rs.NewSecretResource(name, apiTokenResourceType, key.ID, secretTraitOptions) if err != nil { return nil, err } - return &connectorbuilder.CredentialIssueOutput{Secret: secret, PlaintextData: []*v2.PlaintextData{v2.PlaintextData_builder{Name: "api_key", Bytes: []byte(key.Secret)}.Build()}, ResourceMode: v2.CredentialResourceMode_CREDENTIAL_RESOURCE_MODE_DISCOVERABLE}, nil + return &connectorbuilder.CredentialIssueOutput{ + Secret: secret, + PlaintextData: []*v2.PlaintextData{ + v2.PlaintextData_builder{Name: "api_key", Bytes: []byte(key.Secret)}.Build(), + }, + ResourceMode: v2.CredentialResourceMode_CREDENTIAL_RESOURCE_MODE_DISCOVERABLE, + }, nil } var _ connectorbuilder.ResourceSyncerV2 = &userBuilder{} From 062e16c9f3877cda2a734b37f39dba86be96d106 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Bala Krishnan Date: Thu, 20 Aug 2026 05:42:29 +0000 Subject: [PATCH 03/49] Add Datadog credential smoke test Co-authored-by: c1-squire-dev[bot] --- pkg/connector/credential_smoke_test.go | 78 ++++++++++++++++++++++++++ 1 file changed, 78 insertions(+) create mode 100644 pkg/connector/credential_smoke_test.go diff --git a/pkg/connector/credential_smoke_test.go b/pkg/connector/credential_smoke_test.go new file mode 100644 index 00000000..f61cea12 --- /dev/null +++ b/pkg/connector/credential_smoke_test.go @@ -0,0 +1,78 @@ +package connector + +import ( + "context" + "os" + "testing" + "time" + + cfg "github.com/conductorone/baton-datadog/pkg/config" + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/stretchr/testify/require" +) + +// TestCredentialIssueLifecycle is an opt-in live-provider smoke test. It creates +// a real Datadog API key and always attempts to revoke it before returning. +// Run it only in a disposable Datadog organization: +// +// DATADOG_CREDENTIAL_SMOKE=1 BATON_SITE=datadoghq.com BATON_API_KEY=... BATON_APP_KEY=... \ +// go test ./pkg/connector -run TestCredentialIssueLifecycle -count=1 +func TestCredentialIssueLifecycle(t *testing.T) { + if os.Getenv("DATADOG_CREDENTIAL_SMOKE") != "1" { + t.Skip("set DATADOG_CREDENTIAL_SMOKE=1 to run against Datadog") + } + + site := os.Getenv("BATON_SITE") + apiKey := os.Getenv("BATON_API_KEY") + appKey := os.Getenv("BATON_APP_KEY") + require.NotEmpty(t, site, "BATON_SITE is required") + require.NotEmpty(t, apiKey, "BATON_API_KEY is required") + require.NotEmpty(t, appKey, "BATON_APP_KEY is required") + + ctx := context.Background() + builder, _, err := New(ctx, &cfg.Datadog{ + Site: site, + ApiKey: apiKey, + AppKey: appKey, + SyncSecrets: true, + }, nil) + require.NoError(t, err) + datadogConnector, ok := builder.(*Datadog) + require.True(t, ok) + + issuer := newCredentialUserBuilder(datadogConnector.wrapper) + requestID := "smoke-" + time.Now().UTC().Format("20060102T150405") + issued, err := issuer.Issue(ctx, &connectorbuilder.CredentialIssueInput{ + IdentityID: &v2.ResourceId{ + ResourceType: userResourceType.Id, + Resource: "credential-smoke-test", + }, + RequestID: requestID, + }) + require.NoError(t, err) + require.NotNil(t, issued.Secret) + require.NotEmpty(t, issued.Secret.GetId().GetResource()) + require.Len(t, issued.PlaintextData, 1) + require.NotEmpty(t, issued.PlaintextData[0].GetBytes()) + + secretID := issued.Secret.GetId() + deleted := false + t.Cleanup(func() { + if deleted { + return + } + _, deleteErr := newApiTokenBuilder(datadogConnector.wrapper).Delete(ctx, secretID, nil) + require.NoError(t, deleteErr, "Datadog API key cleanup failed: %s", secretID.GetResource()) + }) + + _, err = newApiTokenBuilder(datadogConnector.wrapper).Delete(ctx, secretID, nil) + require.NoError(t, err, "revoke issued Datadog API key") + deleted = true + + keys, err := datadogConnector.wrapper.ListAPIKeys(ctx, nil) + require.NoError(t, err, "list API keys after revocation") + for _, key := range keys.GetData() { + require.NotEqual(t, secretID.GetResource(), key.GetId(), "revoked API key is still listed") + } +} From e9936d379e1110a2ae2105b4c7c1c092c652c040 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Bala Krishnan Date: Thu, 20 Aug 2026 06:57:05 +0000 Subject: [PATCH 04/49] Log Datadog credential smoke lifecycle Co-authored-by: c1-squire-dev[bot] --- pkg/client/client.go | 13 +++++++++++++ pkg/connector/credential_smoke_test.go | 23 +++++++++++++++++++---- 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index a77bc779..960e9d51 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -202,6 +202,19 @@ func (w *DatadogClient) CreateAPIKey(ctx context.Context, name string) (*IssuedA return &IssuedAPIKey{ID: *response.Data.Id, Secret: *response.Data.Attributes.Key}, nil } +func (w *DatadogClient) GetAPIKey(ctx context.Context, id string) (*datadogV2.APIKeyResponse, error) { + ctx = w.withAuthContext(ctx) + api := datadogV2.NewKeyManagementApi(w.officialClient) + response, httpRes, err := api.GetAPIKey(ctx, id) + if httpRes != nil { + defer httpRes.Body.Close() + } + if err != nil { + return nil, wrapOfficialClientError("get API key", httpRes, err) + } + return &response, nil +} + func (w *DatadogClient) DeleteAPIKey(ctx context.Context, id string) error { ctx = w.withAuthContext(ctx) api := datadogV2.NewKeyManagementApi(w.officialClient) diff --git a/pkg/connector/credential_smoke_test.go b/pkg/connector/credential_smoke_test.go index f61cea12..57877d5b 100644 --- a/pkg/connector/credential_smoke_test.go +++ b/pkg/connector/credential_smoke_test.go @@ -3,6 +3,7 @@ package connector import ( "context" "os" + "strings" "testing" "time" @@ -43,6 +44,7 @@ func TestCredentialIssueLifecycle(t *testing.T) { issuer := newCredentialUserBuilder(datadogConnector.wrapper) requestID := "smoke-" + time.Now().UTC().Format("20060102T150405") + t.Logf("issuing Datadog API key with request id %q", requestID) issued, err := issuer.Issue(ctx, &connectorbuilder.CredentialIssueInput{ IdentityID: &v2.ResourceId{ ResourceType: userResourceType.Id, @@ -57,6 +59,13 @@ func TestCredentialIssueLifecycle(t *testing.T) { require.NotEmpty(t, issued.PlaintextData[0].GetBytes()) secretID := issued.Secret.GetId() + t.Logf("issued API key id=%s; plaintext material returned but not logged", maskedValue(secretID.GetResource())) + providerKey, err := datadogConnector.wrapper.GetAPIKey(ctx, secretID.GetResource()) + require.NoError(t, err, "read issued API key from Datadog") + providerKeyData := providerKey.GetData() + require.Equal(t, secretID.GetResource(), providerKeyData.GetId()) + t.Logf("confirmed API key id=%s exists in Datadog", maskedValue(secretID.GetResource())) + deleted := false t.Cleanup(func() { if deleted { @@ -66,13 +75,19 @@ func TestCredentialIssueLifecycle(t *testing.T) { require.NoError(t, deleteErr, "Datadog API key cleanup failed: %s", secretID.GetResource()) }) + t.Logf("revoking API key id=%s", maskedValue(secretID.GetResource())) _, err = newApiTokenBuilder(datadogConnector.wrapper).Delete(ctx, secretID, nil) require.NoError(t, err, "revoke issued Datadog API key") deleted = true - keys, err := datadogConnector.wrapper.ListAPIKeys(ctx, nil) - require.NoError(t, err, "list API keys after revocation") - for _, key := range keys.GetData() { - require.NotEqual(t, secretID.GetResource(), key.GetId(), "revoked API key is still listed") + _, err = datadogConnector.wrapper.GetAPIKey(ctx, secretID.GetResource()) + require.Error(t, err, "revoked API key is still retrievable") + t.Logf("confirmed API key id=%s is no longer retrievable from Datadog", maskedValue(secretID.GetResource())) +} + +func maskedValue(value string) string { + if len(value) <= 4 { + return "***" } + return value[:2] + strings.Repeat("*", len(value)-4) + value[len(value)-2:] } From b33217368a8fab92e1ec33ced08be930b2160ad6 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Bala Krishnan Date: Thu, 20 Aug 2026 06:57:24 +0000 Subject: [PATCH 05/49] Assert Datadog credential revocation Co-authored-by: c1-squire-dev[bot] --- pkg/connector/credential_smoke_test.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pkg/connector/credential_smoke_test.go b/pkg/connector/credential_smoke_test.go index 57877d5b..61afc77a 100644 --- a/pkg/connector/credential_smoke_test.go +++ b/pkg/connector/credential_smoke_test.go @@ -11,6 +11,8 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // TestCredentialIssueLifecycle is an opt-in live-provider smoke test. It creates @@ -81,7 +83,7 @@ func TestCredentialIssueLifecycle(t *testing.T) { deleted = true _, err = datadogConnector.wrapper.GetAPIKey(ctx, secretID.GetResource()) - require.Error(t, err, "revoked API key is still retrievable") + require.Equal(t, codes.NotFound, status.Code(err), "revoked API key is still retrievable") t.Logf("confirmed API key id=%s is no longer retrievable from Datadog", maskedValue(secretID.GetResource())) } From 3929e1fcbbb721173dec0b500e28298137cff245 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Bala Krishnan Date: Thu, 20 Aug 2026 06:59:26 +0000 Subject: [PATCH 06/49] Wait for Datadog credential revocation Co-authored-by: c1-squire-dev[bot] --- pkg/connector/credential_smoke_test.go | 30 ++++++++++++++++++++------ 1 file changed, 23 insertions(+), 7 deletions(-) diff --git a/pkg/connector/credential_smoke_test.go b/pkg/connector/credential_smoke_test.go index 61afc77a..74f92916 100644 --- a/pkg/connector/credential_smoke_test.go +++ b/pkg/connector/credential_smoke_test.go @@ -68,23 +68,39 @@ func TestCredentialIssueLifecycle(t *testing.T) { require.Equal(t, secretID.GetResource(), providerKeyData.GetId()) t.Logf("confirmed API key id=%s exists in Datadog", maskedValue(secretID.GetResource())) - deleted := false + revoked := false t.Cleanup(func() { - if deleted { + if revoked { return } _, deleteErr := newApiTokenBuilder(datadogConnector.wrapper).Delete(ctx, secretID, nil) - require.NoError(t, deleteErr, "Datadog API key cleanup failed: %s", secretID.GetResource()) + if status.Code(deleteErr) != codes.NotFound { + require.NoError(t, deleteErr, "Datadog API key cleanup failed: %s", secretID.GetResource()) + } }) t.Logf("revoking API key id=%s", maskedValue(secretID.GetResource())) _, err = newApiTokenBuilder(datadogConnector.wrapper).Delete(ctx, secretID, nil) require.NoError(t, err, "revoke issued Datadog API key") - deleted = true - _, err = datadogConnector.wrapper.GetAPIKey(ctx, secretID.GetResource()) - require.Equal(t, codes.NotFound, status.Code(err), "revoked API key is still retrievable") - t.Logf("confirmed API key id=%s is no longer retrievable from Datadog", maskedValue(secretID.GetResource())) + const verificationTimeout = 30 * time.Second + deadline := time.Now().Add(verificationTimeout) + for { + _, err = datadogConnector.wrapper.GetAPIKey(ctx, secretID.GetResource()) + if status.Code(err) == codes.NotFound { + revoked = true + t.Logf("confirmed API key id=%s is no longer retrievable from Datadog", maskedValue(secretID.GetResource())) + return + } + if err != nil { + require.NoError(t, err, "read issued API key while waiting for revocation") + } + if time.Now().After(deadline) { + t.Fatalf("Datadog API key is still retrievable after %s; find and revoke key named c1-%s", verificationTimeout, requestID) + } + t.Logf("API key id=%s is still retrievable; waiting for Datadog revocation propagation", maskedValue(secretID.GetResource())) + time.Sleep(time.Second) + } } func maskedValue(value string) string { From c5b7223c2f72a8c4a66fd9c3431d3fcf013d353f Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Bala Krishnan Date: Thu, 20 Aug 2026 07:01:57 +0000 Subject: [PATCH 07/49] Verify revoked Datadog keys cannot authenticate Co-authored-by: c1-squire-dev[bot] --- pkg/client/client.go | 21 ++++++++++++++++ pkg/connector/credential_smoke_test.go | 33 +++++++++++++------------- 2 files changed, 37 insertions(+), 17 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 960e9d51..49cc1007 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -215,6 +215,27 @@ func (w *DatadogClient) GetAPIKey(ctx context.Context, id string) (*datadogV2.AP return &response, nil } +func (w *DatadogClient) ValidateAPIKey(ctx context.Context, apiKey string) (bool, error) { + ctx = context.WithValue( + ctx, + datadog.ContextAPIKeys, + map[string]datadog.APIKey{ + "apiKeyAuth": {Key: apiKey}, + "appKeyAuth": {Key: w.appKey}, + }, + ) + ctx = context.WithValue(ctx, datadog.ContextServerVariables, map[string]string{"site": w.site}) + api := datadogV1.NewAuthenticationApi(w.officialClient) + response, httpRes, err := api.Validate(ctx) + if httpRes != nil { + defer httpRes.Body.Close() + } + if err != nil { + return false, wrapOfficialClientError("validate API key", httpRes, err) + } + return response.GetValid(), nil +} + func (w *DatadogClient) DeleteAPIKey(ctx context.Context, id string) error { ctx = w.withAuthContext(ctx) api := datadogV2.NewKeyManagementApi(w.officialClient) diff --git a/pkg/connector/credential_smoke_test.go b/pkg/connector/credential_smoke_test.go index 74f92916..be105e54 100644 --- a/pkg/connector/credential_smoke_test.go +++ b/pkg/connector/credential_smoke_test.go @@ -67,6 +67,10 @@ func TestCredentialIssueLifecycle(t *testing.T) { providerKeyData := providerKey.GetData() require.Equal(t, secretID.GetResource(), providerKeyData.GetId()) t.Logf("confirmed API key id=%s exists in Datadog", maskedValue(secretID.GetResource())) + issuedKeyValid, err := datadogConnector.wrapper.ValidateAPIKey(ctx, string(issued.PlaintextData[0].GetBytes())) + require.NoError(t, err, "authenticate with issued API key") + require.True(t, issuedKeyValid, "issued API key is not accepted by Datadog") + t.Logf("confirmed issued API key id=%s can authenticate with Datadog", maskedValue(secretID.GetResource())) revoked := false t.Cleanup(func() { @@ -82,25 +86,20 @@ func TestCredentialIssueLifecycle(t *testing.T) { t.Logf("revoking API key id=%s", maskedValue(secretID.GetResource())) _, err = newApiTokenBuilder(datadogConnector.wrapper).Delete(ctx, secretID, nil) require.NoError(t, err, "revoke issued Datadog API key") + issuedKeyValid, err = datadogConnector.wrapper.ValidateAPIKey(ctx, string(issued.PlaintextData[0].GetBytes())) + if err == nil { + require.False(t, issuedKeyValid, "revoked API key can still authenticate with Datadog") + } + require.False(t, issuedKeyValid, "revoked API key can still authenticate with Datadog") + t.Logf("confirmed revoked API key id=%s can no longer authenticate with Datadog", maskedValue(secretID.GetResource())) + revoked = true - const verificationTimeout = 30 * time.Second - deadline := time.Now().Add(verificationTimeout) - for { - _, err = datadogConnector.wrapper.GetAPIKey(ctx, secretID.GetResource()) - if status.Code(err) == codes.NotFound { - revoked = true - t.Logf("confirmed API key id=%s is no longer retrievable from Datadog", maskedValue(secretID.GetResource())) - return - } - if err != nil { - require.NoError(t, err, "read issued API key while waiting for revocation") - } - if time.Now().After(deadline) { - t.Fatalf("Datadog API key is still retrievable after %s; find and revoke key named c1-%s", verificationTimeout, requestID) - } - t.Logf("API key id=%s is still retrievable; waiting for Datadog revocation propagation", maskedValue(secretID.GetResource())) - time.Sleep(time.Second) + _, err = datadogConnector.wrapper.GetAPIKey(ctx, secretID.GetResource()) + if status.Code(err) == codes.NotFound { + t.Logf("confirmed API key id=%s is no longer retrievable from Datadog", maskedValue(secretID.GetResource())) + return } + t.Logf("API key metadata id=%s remains retrievable after revocation; this does not imply the key can authenticate", maskedValue(secretID.GetResource())) } func maskedValue(value string) string { From dfb333924075038944df78bbadefe508186b91f8 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Bala Krishnan Date: Thu, 20 Aug 2026 07:02:13 +0000 Subject: [PATCH 08/49] Require authentication failure after revocation Co-authored-by: c1-squire-dev[bot] --- pkg/connector/credential_smoke_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/connector/credential_smoke_test.go b/pkg/connector/credential_smoke_test.go index be105e54..7d1e5626 100644 --- a/pkg/connector/credential_smoke_test.go +++ b/pkg/connector/credential_smoke_test.go @@ -87,10 +87,11 @@ func TestCredentialIssueLifecycle(t *testing.T) { _, err = newApiTokenBuilder(datadogConnector.wrapper).Delete(ctx, secretID, nil) require.NoError(t, err, "revoke issued Datadog API key") issuedKeyValid, err = datadogConnector.wrapper.ValidateAPIKey(ctx, string(issued.PlaintextData[0].GetBytes())) - if err == nil { + if err != nil { + require.Contains(t, []codes.Code{codes.Unauthenticated, codes.PermissionDenied}, status.Code(err), "validate revoked API key") + } else { require.False(t, issuedKeyValid, "revoked API key can still authenticate with Datadog") } - require.False(t, issuedKeyValid, "revoked API key can still authenticate with Datadog") t.Logf("confirmed revoked API key id=%s can no longer authenticate with Datadog", maskedValue(secretID.GetResource())) revoked = true From 72b7a904b6ccc07d7c8a979b7780afb39d41ca4c Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Bala Krishnan Date: Thu, 20 Aug 2026 07:03:21 +0000 Subject: [PATCH 09/49] Wait for Datadog key propagation Co-authored-by: c1-squire-dev[bot] --- pkg/client/client.go | 1 - pkg/connector/credential_smoke_test.go | 19 ++++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 49cc1007..407849c3 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -221,7 +221,6 @@ func (w *DatadogClient) ValidateAPIKey(ctx context.Context, apiKey string) (bool datadog.ContextAPIKeys, map[string]datadog.APIKey{ "apiKeyAuth": {Key: apiKey}, - "appKeyAuth": {Key: w.appKey}, }, ) ctx = context.WithValue(ctx, datadog.ContextServerVariables, map[string]string{"site": w.site}) diff --git a/pkg/connector/credential_smoke_test.go b/pkg/connector/credential_smoke_test.go index 7d1e5626..8207a274 100644 --- a/pkg/connector/credential_smoke_test.go +++ b/pkg/connector/credential_smoke_test.go @@ -67,9 +67,11 @@ func TestCredentialIssueLifecycle(t *testing.T) { providerKeyData := providerKey.GetData() require.Equal(t, secretID.GetResource(), providerKeyData.GetId()) t.Logf("confirmed API key id=%s exists in Datadog", maskedValue(secretID.GetResource())) - issuedKeyValid, err := datadogConnector.wrapper.ValidateAPIKey(ctx, string(issued.PlaintextData[0].GetBytes())) - require.NoError(t, err, "authenticate with issued API key") - require.True(t, issuedKeyValid, "issued API key is not accepted by Datadog") + t.Logf("waiting for issued API key id=%s to propagate", maskedValue(secretID.GetResource())) + require.Eventually(t, func() bool { + issuedKeyValid, validateErr := datadogConnector.wrapper.ValidateAPIKey(ctx, string(issued.PlaintextData[0].GetBytes())) + return validateErr == nil && issuedKeyValid + }, 30*time.Second, time.Second, "issued API key did not become usable") t.Logf("confirmed issued API key id=%s can authenticate with Datadog", maskedValue(secretID.GetResource())) revoked := false @@ -86,12 +88,11 @@ func TestCredentialIssueLifecycle(t *testing.T) { t.Logf("revoking API key id=%s", maskedValue(secretID.GetResource())) _, err = newApiTokenBuilder(datadogConnector.wrapper).Delete(ctx, secretID, nil) require.NoError(t, err, "revoke issued Datadog API key") - issuedKeyValid, err = datadogConnector.wrapper.ValidateAPIKey(ctx, string(issued.PlaintextData[0].GetBytes())) - if err != nil { - require.Contains(t, []codes.Code{codes.Unauthenticated, codes.PermissionDenied}, status.Code(err), "validate revoked API key") - } else { - require.False(t, issuedKeyValid, "revoked API key can still authenticate with Datadog") - } + t.Logf("waiting for revoked API key id=%s to stop authenticating", maskedValue(secretID.GetResource())) + require.Eventually(t, func() bool { + issuedKeyValid, validateErr := datadogConnector.wrapper.ValidateAPIKey(ctx, string(issued.PlaintextData[0].GetBytes())) + return !issuedKeyValid && (validateErr == nil || status.Code(validateErr) == codes.Unauthenticated || status.Code(validateErr) == codes.PermissionDenied) + }, 30*time.Second, time.Second, "revoked API key can still authenticate with Datadog") t.Logf("confirmed revoked API key id=%s can no longer authenticate with Datadog", maskedValue(secretID.GetResource())) revoked = true From eac69637484ac71beef71e9ff48c93c2c4595c96 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Bala Krishnan Date: Thu, 20 Aug 2026 07:10:00 +0000 Subject: [PATCH 10/49] Address Datadog credential review feedback Co-authored-by: c1-squire-dev[bot] --- baton_capabilities.json | 21 +++++++++++++- docs/connector.mdx | 10 +++++-- pkg/connector/api_token.go | 5 ++++ pkg/connector/credential_smoke_test.go | 40 +++++++++++++------------- pkg/connector/resource_types.go | 4 ++- pkg/connector/users.go | 9 ++++++ 6 files changed, 64 insertions(+), 25 deletions(-) diff --git a/baton_capabilities.json b/baton_capabilities.json index 21f23acf..fa7c22c3 100644 --- a/baton_capabilities.json +++ b/baton_capabilities.json @@ -17,10 +17,17 @@ "permissions": [ { "permission": "api_keys_read" + }, + { + "permission": "api_keys_write" + }, + { + "permission": "api_keys_delete" } ] } - ] + ], + "description": "Credential issuance creates keys owned by the connector's Datadog principal, not the selected Datadog user." }, "capabilities": [ "CAPABILITY_SYNC", @@ -30,6 +37,12 @@ "permissions": [ { "permission": "api_keys_read" + }, + { + "permission": "api_keys_write" + }, + { + "permission": "api_keys_delete" } ] } @@ -137,6 +150,9 @@ { "@type": "type.googleapis.com/c1.connector.v2.CapabilityPermissions", "permissions": [ + { + "permission": "api_keys_write" + }, { "permission": "user_access_invite" }, @@ -154,6 +170,9 @@ ], "permissions": { "permissions": [ + { + "permission": "api_keys_write" + }, { "permission": "user_access_invite" }, diff --git a/docs/connector.mdx b/docs/connector.mdx index 5ac56d01..ff0b3d3d 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -22,9 +22,13 @@ sidebarTitle: "Datadog" [This connector can sync secrets](/product/admin/inventory) and display them on the **Inventory** page. -API keys can be issued and revoked through C1. Datadog does not support an expiration date when creating an API key. +API keys can be issued and revoked through C1 when **Sync secrets** is enabled. Datadog does not support an expiration date when creating an API key. -*Schedules are not synced by default, but you can opt into syncing them when configuring the connector. + +Datadog creates issued API keys under the connector's authenticated Datadog user or service account. The selected Datadog user is recorded in C1 for tracking, but does not become the provider-side API-key owner or limit the key's provider-side scope. + + +*Schedules and API-key issuance are not enabled by default. Enable **Sync schedules** or **Sync secrets**, respectively, when configuring the connector. ### Connector actions @@ -43,7 +47,7 @@ Configuring the connector requires you to pass in credentials generated in Datad A user with the **Connector Administrator** or **Super Administrator** role in C1 and the **Datadog Admin** or **Datadog standard** role in Datadog must perform this task. -If your user has a custom Datadog role, make sure it includes the **User App Keys** and **API Keys Read** permissions, plus **User Access Invite** and **User Access Manage** to create, update, enable, and disable users from C1. +If your user has a custom Datadog role, make sure it includes the **User App Keys**, **API Keys Read**, **API Keys Write**, and **API Keys Delete** permissions, plus **User Access Invite** and **User Access Manage** to create, update, enable, and disable users from C1. ### Locate your Datadog site diff --git a/pkg/connector/api_token.go b/pkg/connector/api_token.go index 7f55c8c7..5fe4611c 100644 --- a/pkg/connector/api_token.go +++ b/pkg/connector/api_token.go @@ -13,6 +13,8 @@ import ( "github.com/conductorone/baton-sdk/pkg/types/resource" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type apiTokenBuilder struct { @@ -28,6 +30,9 @@ func (o *apiTokenBuilder) Delete(ctx context.Context, resourceID *v2.ResourceId, return nil, fmt.Errorf("baton-datadog: API key id is required") } if err := o.wrapper.DeleteAPIKey(ctx, resourceID.GetResource()); err != nil { + if status.Code(err) == codes.NotFound { + return nil, nil + } return nil, fmt.Errorf("baton-datadog: delete API key: %w", err) } return nil, nil diff --git a/pkg/connector/credential_smoke_test.go b/pkg/connector/credential_smoke_test.go index 8207a274..ae567482 100644 --- a/pkg/connector/credential_smoke_test.go +++ b/pkg/connector/credential_smoke_test.go @@ -19,19 +19,19 @@ import ( // a real Datadog API key and always attempts to revoke it before returning. // Run it only in a disposable Datadog organization: // -// DATADOG_CREDENTIAL_SMOKE=1 BATON_SITE=datadoghq.com BATON_API_KEY=... BATON_APP_KEY=... \ -// go test ./pkg/connector -run TestCredentialIssueLifecycle -count=1 +// DATADOG_CREDENTIAL_SMOKE=1 DATADOG_SMOKE_SITE=datadoghq.com DATADOG_SMOKE_API_KEY=... DATADOG_SMOKE_APP_KEY=... \ +// go test ./pkg/connector -run TestCredentialIssueLifecycle -count=1 func TestCredentialIssueLifecycle(t *testing.T) { if os.Getenv("DATADOG_CREDENTIAL_SMOKE") != "1" { t.Skip("set DATADOG_CREDENTIAL_SMOKE=1 to run against Datadog") } - site := os.Getenv("BATON_SITE") - apiKey := os.Getenv("BATON_API_KEY") - appKey := os.Getenv("BATON_APP_KEY") - require.NotEmpty(t, site, "BATON_SITE is required") - require.NotEmpty(t, apiKey, "BATON_API_KEY is required") - require.NotEmpty(t, appKey, "BATON_APP_KEY is required") + site := os.Getenv("DATADOG_SMOKE_SITE") + apiKey := os.Getenv("DATADOG_SMOKE_API_KEY") + appKey := os.Getenv("DATADOG_SMOKE_APP_KEY") + require.NotEmpty(t, site, "DATADOG_SMOKE_SITE is required") + require.NotEmpty(t, apiKey, "DATADOG_SMOKE_API_KEY is required") + require.NotEmpty(t, appKey, "DATADOG_SMOKE_APP_KEY is required") ctx := context.Background() builder, _, err := New(ctx, &cfg.Datadog{ @@ -55,9 +55,20 @@ func TestCredentialIssueLifecycle(t *testing.T) { RequestID: requestID, }) require.NoError(t, err) + revoked := false + t.Cleanup(func() { + if revoked || issued == nil || issued.Secret == nil || issued.Secret.GetId() == nil || issued.Secret.GetId().GetResource() == "" { + return + } + secretID := issued.Secret.GetId() + _, deleteErr := newApiTokenBuilder(datadogConnector.wrapper).Delete(ctx, secretID, nil) + if status.Code(deleteErr) != codes.NotFound { + require.NoError(t, deleteErr, "Datadog API key cleanup failed: %s", secretID.GetResource()) + } + }) require.NotNil(t, issued.Secret) require.NotEmpty(t, issued.Secret.GetId().GetResource()) - require.Len(t, issued.PlaintextData, 1) + require.Equal(t, 1, len(issued.PlaintextData)) require.NotEmpty(t, issued.PlaintextData[0].GetBytes()) secretID := issued.Secret.GetId() @@ -74,17 +85,6 @@ func TestCredentialIssueLifecycle(t *testing.T) { }, 30*time.Second, time.Second, "issued API key did not become usable") t.Logf("confirmed issued API key id=%s can authenticate with Datadog", maskedValue(secretID.GetResource())) - revoked := false - t.Cleanup(func() { - if revoked { - return - } - _, deleteErr := newApiTokenBuilder(datadogConnector.wrapper).Delete(ctx, secretID, nil) - if status.Code(deleteErr) != codes.NotFound { - require.NoError(t, deleteErr, "Datadog API key cleanup failed: %s", secretID.GetResource()) - } - }) - t.Logf("revoking API key id=%s", maskedValue(secretID.GetResource())) _, err = newApiTokenBuilder(datadogConnector.wrapper).Delete(ctx, secretID, nil) require.NoError(t, err, "revoke issued Datadog API key") diff --git a/pkg/connector/resource_types.go b/pkg/connector/resource_types.go index fa6b84dd..76dadb98 100644 --- a/pkg/connector/resource_types.go +++ b/pkg/connector/resource_types.go @@ -22,6 +22,7 @@ var ( Annotations: annotations.New( &v2.SkipEntitlementsAndGrants{}, capabilityPermissions( + "api_keys_write", "user_access_invite", "user_access_manage", ), @@ -46,10 +47,11 @@ var ( apiTokenResourceType = &v2.ResourceType{ Id: "api-key", DisplayName: "API Key", + Description: "Credential issuance creates keys owned by the connector's Datadog principal, not the selected Datadog user.", Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_SECRET}, Annotations: annotations.New( &v2.SkipEntitlementsAndGrants{}, - capabilityPermissions("api_keys_read"), + capabilityPermissions("api_keys_read", "api_keys_write", "api_keys_delete"), ), } scheduleResourceType = &v2.ResourceType{ diff --git a/pkg/connector/users.go b/pkg/connector/users.go index cb9759ce..20a0a9ec 100644 --- a/pkg/connector/users.go +++ b/pkg/connector/users.go @@ -12,6 +12,8 @@ import ( "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" rs "github.com/conductorone/baton-sdk/pkg/types/resource" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" ) type userBuilder struct { @@ -53,6 +55,12 @@ func (u *credentialUserBuilder) Issue(ctx context.Context, input *connectorbuild } secret, err := rs.NewSecretResource(name, apiTokenResourceType, key.ID, secretTraitOptions) if err != nil { + if deleteErr := u.wrapper.DeleteAPIKey(ctx, key.ID); deleteErr != nil { + ctxzap.Extract(ctx).Warn("failed to clean up Datadog API key after resource construction error", + zap.String("api_key_id", key.ID), + zap.Error(deleteErr), + ) + } return nil, err } return &connectorbuilder.CredentialIssueOutput{ @@ -67,6 +75,7 @@ func (u *credentialUserBuilder) Issue(ctx context.Context, input *connectorbuild var _ connectorbuilder.ResourceSyncerV2 = &userBuilder{} var _ connectorbuilder.AccountManagerV2 = &userBuilder{} var _ connectorbuilder.ResourceActionProvider = &userBuilder{} +var _ connectorbuilder.CredentialIssuerLimited = &credentialUserBuilder{} // ResourceActions registers user-scoped actions. Only update_user lives here: it is // reached through the generic "Perform connector action" step (which supplies a resource From 9fae1c92bea0b3bba80f6fdba9b6128c77fc83b0 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Bala Krishnan Date: Thu, 20 Aug 2026 07:11:10 +0000 Subject: [PATCH 11/49] Guard missing API key response data Co-authored-by: c1-squire-dev[bot] --- pkg/client/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 407849c3..71a7568a 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -196,7 +196,7 @@ func (w *DatadogClient) CreateAPIKey(ctx context.Context, name string) (*IssuedA if err != nil { return nil, wrapOfficialClientError("create API key", httpRes, err) } - if response.Data.Id == nil || response.Data.Attributes == nil || response.Data.Attributes.Key == nil || *response.Data.Attributes.Key == "" { + if response.Data == nil || response.Data.Id == nil || response.Data.Attributes == nil || response.Data.Attributes.Key == nil || *response.Data.Attributes.Key == "" { return nil, fmt.Errorf("create API key response omitted id or key") } return &IssuedAPIKey{ID: *response.Data.Id, Secret: *response.Data.Attributes.Key}, nil From 660a6d6ad9de0fa6bd11d2d315d612752e2200f9 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Bala Krishnan Date: Thu, 20 Aug 2026 07:11:46 +0000 Subject: [PATCH 12/49] Clarify Datadog credential smoke validation Co-authored-by: c1-squire-dev[bot] --- pkg/client/client.go | 18 +++++++++++++----- pkg/connector/credential_smoke_test.go | 6 +++++- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 71a7568a..41b27c36 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -105,9 +105,7 @@ func (w *DatadogClient) ListUsers(ctx context.Context, params *datadogV2.ListUse // Validate validates API credentials using the REST client. func (w *DatadogClient) Validate(ctx context.Context) (*datadogV1.AuthenticationValidationResponse, error) { - // TODO: Implement in DatadogRestClient - // For now, return error indicating this needs to be implemented - return nil, fmt.Errorf("Validate not yet implemented in REST client") + return w.validateAPIKey(w.withAuthContext(ctx)) } // ListTeams lists teams using the REST client. @@ -203,6 +201,7 @@ func (w *DatadogClient) CreateAPIKey(ctx context.Context, name string) (*IssuedA } func (w *DatadogClient) GetAPIKey(ctx context.Context, id string) (*datadogV2.APIKeyResponse, error) { + // GET /api/v2/api_keys/{api_key_id}. Requires the api_keys_read permission. ctx = w.withAuthContext(ctx) api := datadogV2.NewKeyManagementApi(w.officialClient) response, httpRes, err := api.GetAPIKey(ctx, id) @@ -224,15 +223,24 @@ func (w *DatadogClient) ValidateAPIKey(ctx context.Context, apiKey string) (bool }, ) ctx = context.WithValue(ctx, datadog.ContextServerVariables, map[string]string{"site": w.site}) + response, err := w.validateAPIKey(ctx) + if err != nil { + return false, err + } + return response.GetValid(), nil +} + +// validateAPIKey calls GET /api/v1/validate. It requires an API key and does not require an application key. +func (w *DatadogClient) validateAPIKey(ctx context.Context) (*datadogV1.AuthenticationValidationResponse, error) { api := datadogV1.NewAuthenticationApi(w.officialClient) response, httpRes, err := api.Validate(ctx) if httpRes != nil { defer httpRes.Body.Close() } if err != nil { - return false, wrapOfficialClientError("validate API key", httpRes, err) + return nil, wrapOfficialClientError("validate API key", httpRes, err) } - return response.GetValid(), nil + return &response, nil } func (w *DatadogClient) DeleteAPIKey(ctx context.Context, id string) error { diff --git a/pkg/connector/credential_smoke_test.go b/pkg/connector/credential_smoke_test.go index ae567482..17ca5fe0 100644 --- a/pkg/connector/credential_smoke_test.go +++ b/pkg/connector/credential_smoke_test.go @@ -97,11 +97,15 @@ func TestCredentialIssueLifecycle(t *testing.T) { revoked = true _, err = datadogConnector.wrapper.GetAPIKey(ctx, secretID.GetResource()) + if err == nil { + t.Logf("API key metadata id=%s remains retrievable after revocation; this does not imply the key can authenticate", maskedValue(secretID.GetResource())) + return + } if status.Code(err) == codes.NotFound { t.Logf("confirmed API key id=%s is no longer retrievable from Datadog", maskedValue(secretID.GetResource())) return } - t.Logf("API key metadata id=%s remains retrievable after revocation; this does not imply the key can authenticate", maskedValue(secretID.GetResource())) + t.Logf("could not read API key metadata id=%s after revocation: %v", maskedValue(secretID.GetResource()), err) } func maskedValue(value string) string { From 5634acbe81040d35b68db88c1fac94ad47ca79e4 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Bala Krishnan Date: Thu, 20 Aug 2026 08:30:52 +0000 Subject: [PATCH 13/49] Address Datadog credential safety feedback Co-authored-by: c1-squire-dev[bot] --- docs/connector.mdx | 2 +- pkg/client/client.go | 40 +++++++++++++++---------- pkg/client/client_test.go | 53 +++++++++++++++++++++++++++++++++ pkg/connector/resource_types.go | 1 - pkg/connector/users.go | 11 ++++++- 5 files changed, 88 insertions(+), 19 deletions(-) diff --git a/docs/connector.mdx b/docs/connector.mdx index ff0b3d3d..8356cf38 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -47,7 +47,7 @@ Configuring the connector requires you to pass in credentials generated in Datad A user with the **Connector Administrator** or **Super Administrator** role in C1 and the **Datadog Admin** or **Datadog standard** role in Datadog must perform this task. -If your user has a custom Datadog role, make sure it includes the **User App Keys**, **API Keys Read**, **API Keys Write**, and **API Keys Delete** permissions, plus **User Access Invite** and **User Access Manage** to create, update, enable, and disable users from C1. +If your user has a custom Datadog role, make sure it includes **User App Keys**, **User Access Invite**, and **User Access Manage** to create, update, enable, and disable users from C1. If you enable **Sync secrets** to issue or revoke API keys, also add **API Keys Read**, **API Keys Write**, and **API Keys Delete**. ### Locate your Datadog site diff --git a/pkg/client/client.go b/pkg/client/client.go index 41b27c36..dc022f46 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -103,11 +103,6 @@ func (w *DatadogClient) ListUsers(ctx context.Context, params *datadogV2.ListUse return &resp, nil } -// Validate validates API credentials using the REST client. -func (w *DatadogClient) Validate(ctx context.Context) (*datadogV1.AuthenticationValidationResponse, error) { - return w.validateAPIKey(w.withAuthContext(ctx)) -} - // ListTeams lists teams using the REST client. func (w *DatadogClient) ListTeams(ctx context.Context, params *datadogV2.ListTeamsOptionalParameters) (*datadogV2.TeamsResponse, error) { ctx = w.withAuthContext(ctx) @@ -200,6 +195,27 @@ func (w *DatadogClient) CreateAPIKey(ctx context.Context, name string) (*IssuedA return &IssuedAPIKey{ID: *response.Data.Id, Secret: *response.Data.Attributes.Key}, nil } +// FindAPIKeyByName returns an exact name match, if one exists. Datadog's filter +// is a string search, so compare the returned name exactly before treating it as +// an existing issuance. +func (w *DatadogClient) FindAPIKeyByName(ctx context.Context, name string) (*datadogV2.PartialAPIKey, error) { + ctx = w.withAuthContext(ctx) + api := datadogV2.NewKeyManagementApi(w.officialClient) + response, httpRes, err := api.ListAPIKeys(ctx, *datadogV2.NewListAPIKeysOptionalParameters().WithFilter(name).WithPageSize(100)) + if httpRes != nil { + defer httpRes.Body.Close() + } + if err != nil { + return nil, wrapOfficialClientError("find API key by name", httpRes, err) + } + for _, key := range response.GetData() { + if key.Attributes != nil && key.Attributes.GetName() == name { + return &key, nil + } + } + return nil, nil +} + func (w *DatadogClient) GetAPIKey(ctx context.Context, id string) (*datadogV2.APIKeyResponse, error) { // GET /api/v2/api_keys/{api_key_id}. Requires the api_keys_read permission. ctx = w.withAuthContext(ctx) @@ -223,24 +239,16 @@ func (w *DatadogClient) ValidateAPIKey(ctx context.Context, apiKey string) (bool }, ) ctx = context.WithValue(ctx, datadog.ContextServerVariables, map[string]string{"site": w.site}) - response, err := w.validateAPIKey(ctx) - if err != nil { - return false, err - } - return response.GetValid(), nil -} - -// validateAPIKey calls GET /api/v1/validate. It requires an API key and does not require an application key. -func (w *DatadogClient) validateAPIKey(ctx context.Context) (*datadogV1.AuthenticationValidationResponse, error) { + // GET /api/v1/validate requires an API key and does not require an application key. api := datadogV1.NewAuthenticationApi(w.officialClient) response, httpRes, err := api.Validate(ctx) if httpRes != nil { defer httpRes.Body.Close() } if err != nil { - return nil, wrapOfficialClientError("validate API key", httpRes, err) + return false, wrapOfficialClientError("validate API key", httpRes, err) } - return &response, nil + return response.GetValid(), nil } func (w *DatadogClient) DeleteAPIKey(ctx context.Context, id string) error { diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 4a329432..dfdce204 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -9,6 +9,10 @@ import ( "strings" "testing" "time" + + "github.com/DataDog/datadog-api-client-go/v2/api/datadog" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // Helper function to check if two strings are equal. @@ -19,6 +23,55 @@ func assertEqual(t *testing.T, expected, actual string, message string) { } } +func newOfficialTestClient(serverURL string) *DatadogClient { + cfg := datadog.NewConfiguration() + cfg.Servers = datadog.ServerConfigurations{{URL: serverURL}} + return NewDatadogClient(nil, datadog.NewAPIClient(cfg), "example.com", testAPIKey, testAppKey) +} + +func TestAPIKeyManagement(t *testing.T) { + t.Run("create returns issued material", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assertEqual(t, http.MethodPost, r.Method, "HTTP method should match") + assertEqual(t, "/api/v2/api_keys", r.URL.Path, "request path should match") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":{"id":"key-id","type":"api_keys","attributes":{"key":"plaintext-key","name":"c1-request"}}}`)) + })) + defer server.Close() + + issued, err := newOfficialTestClient(server.URL).CreateAPIKey(context.Background(), "c1-request") + assertNoError(t, err, "create API key should succeed") + assertEqual(t, "key-id", issued.ID, "issued key ID should match") + assertEqual(t, "plaintext-key", issued.Secret, "issued key material should match") + }) + + t.Run("create rejects a response without plaintext material", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":{"id":"key-id","type":"api_keys","attributes":{}}}`)) + })) + defer server.Close() + + _, err := newOfficialTestClient(server.URL).CreateAPIKey(context.Background(), "c1-request") + assertError(t, err, "create API key should reject missing plaintext material") + }) + + t.Run("delete maps a provider 404 to not found", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assertEqual(t, http.MethodDelete, r.Method, "HTTP method should match") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":["Not found"]}`)) + })) + defer server.Close() + + err := newOfficialTestClient(server.URL).DeleteAPIKey(context.Background(), "missing-key") + if status.Code(err) != codes.NotFound { + t.Fatalf("DeleteAPIKey() error code = %s, want %s; error = %v", status.Code(err), codes.NotFound, err) + } + }) +} + // Helper function to check if a value is not nil. func assertNotNil(t *testing.T, value interface{}, message string) { t.Helper() diff --git a/pkg/connector/resource_types.go b/pkg/connector/resource_types.go index 76dadb98..e387e4e9 100644 --- a/pkg/connector/resource_types.go +++ b/pkg/connector/resource_types.go @@ -22,7 +22,6 @@ var ( Annotations: annotations.New( &v2.SkipEntitlementsAndGrants{}, capabilityPermissions( - "api_keys_write", "user_access_invite", "user_access_manage", ), diff --git a/pkg/connector/users.go b/pkg/connector/users.go index 20a0a9ec..49716598 100644 --- a/pkg/connector/users.go +++ b/pkg/connector/users.go @@ -43,6 +43,13 @@ func (u *credentialUserBuilder) Issue(ctx context.Context, input *connectorbuild return nil, fmt.Errorf("baton-datadog: a Datadog user identity is required") } name := "c1-" + input.RequestID + existing, err := u.wrapper.FindAPIKeyByName(ctx, name) + if err != nil { + return nil, fmt.Errorf("baton-datadog: look up API key for request %q: %w", input.RequestID, err) + } + if existing != nil { + return nil, fmt.Errorf("baton-datadog: API key for request %q may already exist; refusing to issue a duplicate", input.RequestID) + } key, err := u.wrapper.CreateAPIKey(ctx, name) if err != nil { return nil, fmt.Errorf("baton-datadog: create API key: %w", err) @@ -61,7 +68,7 @@ func (u *credentialUserBuilder) Issue(ctx context.Context, input *connectorbuild zap.Error(deleteErr), ) } - return nil, err + return nil, fmt.Errorf("baton-datadog: build API key secret resource: %w", err) } return &connectorbuilder.CredentialIssueOutput{ Secret: secret, @@ -76,6 +83,8 @@ var _ connectorbuilder.ResourceSyncerV2 = &userBuilder{} var _ connectorbuilder.AccountManagerV2 = &userBuilder{} var _ connectorbuilder.ResourceActionProvider = &userBuilder{} var _ connectorbuilder.CredentialIssuerLimited = &credentialUserBuilder{} +var _ connectorbuilder.AccountManagerV2 = &credentialUserBuilder{} +var _ connectorbuilder.ResourceActionProvider = &credentialUserBuilder{} // ResourceActions registers user-scoped actions. Only update_user lives here: it is // reached through the generic "Perform connector action" step (which supplies a resource From c1e8e8583f4a5d147fe8e3826d5479813e0cd125 Mon Sep 17 00:00:00 2001 From: Santhosh Kumar Bala Krishnan Date: Thu, 20 Aug 2026 08:33:39 +0000 Subject: [PATCH 14/49] Regenerate baton_capabilities.json The user resource type stopped declaring api_keys_write in 5634acb; the committed metadata still carried it, failing validate_metadata. Co-authored-by: c1-squire-dev[bot] --- baton_capabilities.json | 6 ------ 1 file changed, 6 deletions(-) diff --git a/baton_capabilities.json b/baton_capabilities.json index fa7c22c3..992d5da0 100644 --- a/baton_capabilities.json +++ b/baton_capabilities.json @@ -150,9 +150,6 @@ { "@type": "type.googleapis.com/c1.connector.v2.CapabilityPermissions", "permissions": [ - { - "permission": "api_keys_write" - }, { "permission": "user_access_invite" }, @@ -170,9 +167,6 @@ ], "permissions": { "permissions": [ - { - "permission": "api_keys_write" - }, { "permission": "user_access_invite" }, From 020525dc52f00bc35616a9c7ddeb1b76d06b8602 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:18:20 +0000 Subject: [PATCH 15/49] test: add offline credential-lifecycle regression tests for PR #40 Closes four blocking gaps identified by prior live validation: delete-by-handle secret isolation, malformed/missing handle rejection, handle/secret separation at the issuer boundary, and a secret-log assertion across Issue+Delete. --- pkg/connector/credential_lifecycle_test.go | 209 +++++++++++++++++++++ 1 file changed, 209 insertions(+) create mode 100644 pkg/connector/credential_lifecycle_test.go diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go new file mode 100644 index 00000000..36fb29af --- /dev/null +++ b/pkg/connector/credential_lifecycle_test.go @@ -0,0 +1,209 @@ +package connector + +import ( + "bytes" + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/DataDog/datadog-api-client-go/v2/api/datadog" + "github.com/conductorone/baton-datadog/pkg/client" + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" +) + +// newCredentialLifecycleServer stands in for Datadog across an Issue+Delete +// round trip. It answers FindAPIKeyByName (GET, empty match), CreateAPIKey +// (POST, returns handle+secret), and DeleteAPIKey (DELETE by handle). Every +// request the connector actually sends is recorded so tests can assert on it. +type recordedRequest struct { + method string + path string + query string + header http.Header + body string +} + +func newCredentialLifecycleServer(t *testing.T, handle, secret, name string) (*httptest.Server, *[]recordedRequest) { + t.Helper() + requests := &[]recordedRequest{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + bodyBytes := make([]byte, 0) + if r.Body != nil { + buf := new(bytes.Buffer) + _, _ = buf.ReadFrom(r.Body) + bodyBytes = buf.Bytes() + } + *requests = append(*requests, recordedRequest{ + method: r.Method, + path: r.URL.Path, + query: r.URL.RawQuery, + header: r.Header.Clone(), + body: string(bodyBytes), + }) + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/api_keys": + _, _ = w.Write([]byte(`{"data":[]}`)) + case r.Method == http.MethodPost && r.URL.Path == "/api/v2/api_keys": + _, _ = w.Write([]byte(`{"data":{"id":"` + handle + `","type":"api_keys","attributes":{"key":"` + secret + `","name":"` + name + `"}}}`)) + case r.Method == http.MethodDelete && r.URL.Path == "/api/v2/api_keys/"+handle: + w.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected provider request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + } + })) + return server, requests +} + +func newLifecycleTestWrapper(serverURL string) *client.DatadogClient { + cfg := datadog.NewConfiguration() + cfg.Servers = datadog.ServerConfigurations{{URL: serverURL}} + return client.NewDatadogClient(nil, datadog.NewAPIClient(cfg), "example.com", "connector-api-key", "connector-app-key") +} + +func issueTestCredential(t *testing.T, ctx context.Context, wrapper *client.DatadogClient) *connectorbuilder.CredentialIssueOutput { + t.Helper() + issuer := newCredentialUserBuilder(wrapper) + out, err := issuer.Issue(ctx, &connectorbuilder.CredentialIssueInput{ + IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: "user-1"}, + RequestID: "req-1", + }) + require.NoError(t, err) + return out +} + +// (a) Connector-level delete-by-handle: apiTokenBuilder.Delete must issue the +// provider DELETE for the resource handle, and the plaintext secret must +// never appear anywhere in that request (path, query, headers, or body). +func TestApiTokenBuilderDeleteUsesHandleNotSecret(t *testing.T) { + const ( + handle = "handle-abc123" + secret = "super-secret-plaintext-value" + name = "c1-req-1" + ) + server, requests := newCredentialLifecycleServer(t, handle, secret, name) + defer server.Close() + wrapper := newLifecycleTestWrapper(server.URL) + ctx := context.Background() + + // Issue first so the fake provider has a real handle/secret pair on record, + // then delete strictly by handle -- the way the connector actually calls it. + issued := issueTestCredential(t, ctx, wrapper) + require.Equal(t, handle, issued.Secret.GetId().GetResource()) + require.Equal(t, secret, string(issued.PlaintextData[0].GetBytes())) + + deleter := newApiTokenBuilder(wrapper) + _, err := deleter.Delete(ctx, issued.Secret.GetId(), nil) + require.NoError(t, err) + + var deleteReq *recordedRequest + for i := range *requests { + if (*requests)[i].method == http.MethodDelete { + deleteReq = &(*requests)[i] + } + } + require.NotNil(t, deleteReq, "expected a DELETE request to reach the provider") + require.Equal(t, "/api/v2/api_keys/"+handle, deleteReq.path) + + if strings.Contains(deleteReq.path, secret) || strings.Contains(deleteReq.query, secret) || strings.Contains(deleteReq.body, secret) { + t.Fatalf("delete request leaked the plaintext secret: %+v", deleteReq) + } + for name, values := range deleteReq.header { + for _, v := range values { + if strings.Contains(v, secret) { + t.Fatalf("delete request header %q leaked the plaintext secret: %q", name, v) + } + } + } +} + +// (b) Malformed/missing handle must fail closed before any provider request. +// Only nil ResourceId and an empty ResourceId.Resource are validated by +// pkg/connector/api_token.go:29-31; both are exercised here. +func TestApiTokenBuilderDeleteRejectsMissingHandle(t *testing.T) { + tests := []struct { + name string + resourceID *v2.ResourceId + }{ + {name: "nil ResourceId", resourceID: nil}, + {name: "empty ResourceId.Resource", resourceID: &v2.ResourceId{ResourceType: apiTokenResourceType.Id, Resource: ""}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("provider should not be contacted for a %s, got %s %s", tt.name, r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + wrapper := newLifecycleTestWrapper(server.URL) + + deleter := newApiTokenBuilder(wrapper) + _, err := deleter.Delete(context.Background(), tt.resourceID, nil) + require.Error(t, err) + }) + } +} + +// (c) Handle/secret separation regression at the issuer/connector boundary: +// the returned secret resource ID must equal the provider handle, and must +// never equal the plaintext secret bytes. +func TestIssueHandleAndSecretAreDistinct(t *testing.T) { + const ( + handle = "handle-distinct-1" + secret = "plaintext-distinct-secret" + name = "c1-req-1" + ) + server, _ := newCredentialLifecycleServer(t, handle, secret, name) + defer server.Close() + wrapper := newLifecycleTestWrapper(server.URL) + + issued := issueTestCredential(t, context.Background(), wrapper) + + secretResourceID := issued.Secret.GetId().GetResource() + plaintext := string(issued.PlaintextData[0].GetBytes()) + + require.NotEmpty(t, secretResourceID) + require.NotEmpty(t, plaintext) + require.NotEqual(t, plaintext, secretResourceID, "the secret resource id must not be (or equal) the plaintext secret") + require.Equal(t, handle, secretResourceID, "the secret resource id must equal the provider-issued handle") +} + +// (d) Secret-log assertion: capture every log record emitted across a full +// Issue + Delete cycle and assert the synthetic plaintext secret never +// appears in them. +func TestIssueAndDeleteNeverLogSecret(t *testing.T) { + const ( + handle = "handle-logtest-1" + secret = "plaintext-should-never-be-logged" + name = "c1-req-1" + ) + server, _ := newCredentialLifecycleServer(t, handle, secret, name) + defer server.Close() + wrapper := newLifecycleTestWrapper(server.URL) + + var logBuf bytes.Buffer + encoderCfg := zap.NewProductionEncoderConfig() + core := zapcore.NewCore(zapcore.NewJSONEncoder(encoderCfg), zapcore.AddSync(&logBuf), zapcore.DebugLevel) + logger := zap.New(core) + ctx := ctxzap.ToContext(context.Background(), logger) + + issued := issueTestCredential(t, ctx, wrapper) + require.Equal(t, secret, string(issued.PlaintextData[0].GetBytes())) + + deleter := newApiTokenBuilder(wrapper) + _, err := deleter.Delete(ctx, issued.Secret.GetId(), nil) + require.NoError(t, err) + require.NoError(t, logger.Sync()) + + if strings.Contains(logBuf.String(), secret) { + t.Fatalf("captured logs leaked the plaintext secret:\n%s", logBuf.String()) + } +} From 341e22da57cf5a0914eaa33dc660370a5f072875 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 21 Aug 2026 17:42:41 +0000 Subject: [PATCH 16/49] fix: reject malformed API key handles before deleting apiTokenBuilder.Delete only rejected a nil ResourceId or an empty Resource string, so a non-empty but malformed handle (whitespace-only, or containing a control character) would still reach Datadog's DELETE endpoint. Validate the handle's shape before making the provider call: reject anything empty after trimming or containing a control character. This intentionally does not enforce a stricter format (e.g. UUID) because the vendored Datadog v2 client treats the id as an opaque string with no documented format constraint, and rejecting a handle Datadog considers valid would break real deletes. Co-authored-by: c1-squire-dev[bot] --- pkg/connector/api_token.go | 28 ++++++++++++++++++++-- pkg/connector/credential_lifecycle_test.go | 7 ++++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/pkg/connector/api_token.go b/pkg/connector/api_token.go index 5fe4611c..cc945a16 100644 --- a/pkg/connector/api_token.go +++ b/pkg/connector/api_token.go @@ -3,6 +3,7 @@ package connector import ( "context" "fmt" + "strings" "time" "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" @@ -26,10 +27,14 @@ var _ connectorbuilder.ResourceSyncerV2 = &apiTokenBuilder{} var _ connectorbuilder.ResourceDeleterV2Limited = &apiTokenBuilder{} func (o *apiTokenBuilder) Delete(ctx context.Context, resourceID *v2.ResourceId, _ *v2.ResourceId) (annotations.Annotations, error) { - if resourceID == nil || resourceID.GetResource() == "" { + if resourceID == nil { return nil, fmt.Errorf("baton-datadog: API key id is required") } - if err := o.wrapper.DeleteAPIKey(ctx, resourceID.GetResource()); err != nil { + handle := resourceID.GetResource() + if isMalformedAPIKeyHandle(handle) { + return nil, fmt.Errorf("baton-datadog: API key id %q is malformed", handle) + } + if err := o.wrapper.DeleteAPIKey(ctx, handle); err != nil { if status.Code(err) == codes.NotFound { return nil, nil } @@ -38,6 +43,25 @@ func (o *apiTokenBuilder) Delete(ctx context.Context, resourceID *v2.ResourceId, return nil, nil } +// isMalformedAPIKeyHandle reports whether handle cannot be a valid Datadog API +// key id: empty (including whitespace-only), or containing a control +// character that has no place in an id and is unsafe to embed in a request +// path or log line. The vendored v2 client passes this id through as an +// opaque string with no documented length or charset constraint, so this +// deliberately stays conservative instead of enforcing e.g. UUID shape -- +// rejecting a handle Datadog considers valid would break real deletes. +func isMalformedAPIKeyHandle(handle string) bool { + if strings.TrimSpace(handle) == "" { + return true + } + for _, r := range handle { + if r < 0x20 || r == 0x7f { + return true + } + } + return false +} + func (o *apiTokenBuilder) Entitlements(_ context.Context, _ *v2.Resource, _ resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) { // API Token secrets do not have entitlements return nil, nil, nil diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index 36fb29af..438ea652 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -126,8 +126,9 @@ func TestApiTokenBuilderDeleteUsesHandleNotSecret(t *testing.T) { } // (b) Malformed/missing handle must fail closed before any provider request. -// Only nil ResourceId and an empty ResourceId.Resource are validated by -// pkg/connector/api_token.go:29-31; both are exercised here. +// nil ResourceId, an empty ResourceId.Resource, and a non-empty malformed +// handle (whitespace-only or containing a control character) are all +// validated by pkg/connector/api_token.go:29-36 / isMalformedAPIKeyHandle. func TestApiTokenBuilderDeleteRejectsMissingHandle(t *testing.T) { tests := []struct { name string @@ -135,6 +136,8 @@ func TestApiTokenBuilderDeleteRejectsMissingHandle(t *testing.T) { }{ {name: "nil ResourceId", resourceID: nil}, {name: "empty ResourceId.Resource", resourceID: &v2.ResourceId{ResourceType: apiTokenResourceType.Id, Resource: ""}}, + {name: "whitespace-only handle", resourceID: &v2.ResourceId{ResourceType: apiTokenResourceType.Id, Resource: " "}}, + {name: "handle with control character", resourceID: &v2.ResourceId{ResourceType: apiTokenResourceType.Id, Resource: "handle-\n123"}}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { From 33db92dd0cc0e4338db6e9f28acd7ea42036f5ee Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 21 Aug 2026 18:22:21 +0000 Subject: [PATCH 17/49] fix: use gRPC status codes for hand-rolled credential errors; test FindAPIKeyByName branches; fix testify go.mod marker apiTokenBuilder.Delete and credentialUserBuilder.Issue returned plain errors for their own input-validation rejections (nil/malformed handle, missing identity, duplicate-issuance refusal), so callers only ever saw codes.Unknown even though the connector already uses status.Error(codes.InvalidArgument, ...) for this exact kind of hand-rolled rejection elsewhere (see userIDFromArgs / userIDFromResourceIDArg in actions.go). Follow that existing convention: InvalidArgument for the malformed/missing-identity cases, AlreadyExists for the duplicate-issuance refusal, since that is the standard gRPC code for a conflicting-resource condition and nothing in this repo already claims InvalidArgument for it. Errors that wrap an underlying client error via %w were left alone, since that error already carries a code from wrapOfficialClientError. Added client-level tests for FindAPIKeyByName's two previously untested branches: an exact name match among partial matches, and no match despite a partial one. Added a connector-level test for the consumer of that non-nil branch: Issue must refuse with AlreadyExists and never call CreateAPIKey when a key for the request already exists. go.mod listed testify as indirect even though tests import it directly; `go mod tidy` moved it to the direct require block. go.sum and vendor/ are unchanged (testify was already fully vendored and its vendor/modules.txt entry was already marked explicit), so this is a one-line go.mod correction with no vendor churn. Co-authored-by: c1-squire-dev[bot] --- go.mod | 2 +- pkg/client/client_test.go | 34 +++++++++++++++++++ pkg/connector/api_token.go | 4 +-- pkg/connector/credential_lifecycle_test.go | 38 ++++++++++++++++++++++ pkg/connector/users.go | 6 ++-- 5 files changed, 79 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 6741bdb9..5cd49671 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/ennyjfrick/ruleguard-logfatal v0.0.2 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/quasilyte/go-ruleguard/dsl v0.3.23 + github.com/stretchr/testify v1.11.1 go.uber.org/zap v1.28.0 google.golang.org/grpc v1.83.0 google.golang.org/protobuf v1.36.11 @@ -107,7 +108,6 @@ require ( github.com/spf13/cobra v1.10.2 // indirect github.com/spf13/pflag v1.0.10 // indirect github.com/spf13/viper v1.19.0 // indirect - github.com/stretchr/testify v1.11.1 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/tklauser/go-sysconf v0.3.16 // indirect github.com/tklauser/numcpus v0.11.0 // indirect diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index dfdce204..d67261a0 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -56,6 +56,40 @@ func TestAPIKeyManagement(t *testing.T) { assertError(t, err, "create API key should reject missing plaintext material") }) + t.Run("find by name returns the exact match", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assertEqual(t, http.MethodGet, r.Method, "HTTP method should match") + assertEqual(t, "/api/v2/api_keys", r.URL.Path, "request path should match") + w.Header().Set("Content-Type", "application/json") + // Datadog's filter is a substring search, so the response can contain a + // partial match alongside the exact one; only the exact name should win. + _, _ = w.Write([]byte(`{"data":[ + {"id":"key-partial","type":"api_keys","attributes":{"name":"c1-request-old"}}, + {"id":"key-exact","type":"api_keys","attributes":{"name":"c1-request"}} + ]}`)) + })) + defer server.Close() + + found, err := newOfficialTestClient(server.URL).FindAPIKeyByName(context.Background(), "c1-request") + assertNoError(t, err, "find API key by name should succeed") + assertNotNil(t, found, "expected an exact match") + assertEqual(t, "key-exact", found.GetId(), "exact match should be the id whose name matches exactly") + }) + + t.Run("find by name ignores a non-exact partial match", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[{"id":"key-partial","type":"api_keys","attributes":{"name":"c1-request-old"}}]}`)) + })) + defer server.Close() + + found, err := newOfficialTestClient(server.URL).FindAPIKeyByName(context.Background(), "c1-request") + assertNoError(t, err, "find API key by name should succeed even with no exact match") + if found != nil { + t.Fatalf("expected no exact match, got %+v", found) + } + }) + t.Run("delete maps a provider 404 to not found", func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assertEqual(t, http.MethodDelete, r.Method, "HTTP method should match") diff --git a/pkg/connector/api_token.go b/pkg/connector/api_token.go index cc945a16..26a2ef72 100644 --- a/pkg/connector/api_token.go +++ b/pkg/connector/api_token.go @@ -28,11 +28,11 @@ var _ connectorbuilder.ResourceDeleterV2Limited = &apiTokenBuilder{} func (o *apiTokenBuilder) Delete(ctx context.Context, resourceID *v2.ResourceId, _ *v2.ResourceId) (annotations.Annotations, error) { if resourceID == nil { - return nil, fmt.Errorf("baton-datadog: API key id is required") + return nil, status.Error(codes.InvalidArgument, "baton-datadog: API key id is required") } handle := resourceID.GetResource() if isMalformedAPIKeyHandle(handle) { - return nil, fmt.Errorf("baton-datadog: API key id %q is malformed", handle) + return nil, status.Errorf(codes.InvalidArgument, "baton-datadog: API key id %q is malformed", handle) } if err := o.wrapper.DeleteAPIKey(ctx, handle); err != nil { if status.Code(err) == codes.NotFound { diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index 438ea652..61e69c9e 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -16,6 +16,8 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/zap" "go.uber.org/zap/zapcore" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // newCredentialLifecycleServer stands in for Datadog across an Issue+Delete @@ -210,3 +212,39 @@ func TestIssueAndDeleteNeverLogSecret(t *testing.T) { t.Fatalf("captured logs leaked the plaintext secret:\n%s", logBuf.String()) } } + +// TestIssueRefusesDuplicateRequest exercises the FindAPIKeyByName exact-match +// branch as consumed by Issue: when the provider already has a key named for +// this request, Issue must refuse with AlreadyExists and must never call +// CreateAPIKey (POST /api/v2/api_keys). +func TestIssueRefusesDuplicateRequest(t *testing.T) { + const ( + requestID = "req-dup-1" + name = "c1-" + requestID + existingID = "handle-existing-1" + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/api_keys": + _, _ = w.Write([]byte(`{"data":[{"id":"` + existingID + `","type":"api_keys","attributes":{"name":"` + name + `"}}]}`)) + case r.Method == http.MethodPost && r.URL.Path == "/api/v2/api_keys": + t.Errorf("CreateAPIKey should not be called when a key for this request already exists") + w.WriteHeader(http.StatusInternalServerError) + default: + t.Errorf("unexpected provider request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + } + })) + defer server.Close() + wrapper := newLifecycleTestWrapper(server.URL) + + issuer := newCredentialUserBuilder(wrapper) + out, err := issuer.Issue(context.Background(), &connectorbuilder.CredentialIssueInput{ + IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: "user-1"}, + RequestID: requestID, + }) + require.Nil(t, out) + require.Error(t, err) + require.Equal(t, codes.AlreadyExists, status.Code(err)) +} diff --git a/pkg/connector/users.go b/pkg/connector/users.go index 49716598..1238165f 100644 --- a/pkg/connector/users.go +++ b/pkg/connector/users.go @@ -14,6 +14,8 @@ import ( rs "github.com/conductorone/baton-sdk/pkg/types/resource" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) type userBuilder struct { @@ -40,7 +42,7 @@ func (u *credentialUserBuilder) IssueCapabilityDetails(context.Context) (*v2.Cre func (u *credentialUserBuilder) Issue(ctx context.Context, input *connectorbuilder.CredentialIssueInput) (*connectorbuilder.CredentialIssueOutput, error) { if input == nil || input.IdentityID == nil || input.IdentityID.GetResourceType() != userResourceType.Id { - return nil, fmt.Errorf("baton-datadog: a Datadog user identity is required") + return nil, status.Error(codes.InvalidArgument, "baton-datadog: a Datadog user identity is required") } name := "c1-" + input.RequestID existing, err := u.wrapper.FindAPIKeyByName(ctx, name) @@ -48,7 +50,7 @@ func (u *credentialUserBuilder) Issue(ctx context.Context, input *connectorbuild return nil, fmt.Errorf("baton-datadog: look up API key for request %q: %w", input.RequestID, err) } if existing != nil { - return nil, fmt.Errorf("baton-datadog: API key for request %q may already exist; refusing to issue a duplicate", input.RequestID) + return nil, status.Errorf(codes.AlreadyExists, "baton-datadog: API key for request %q may already exist; refusing to issue a duplicate", input.RequestID) } key, err := u.wrapper.CreateAPIKey(ctx, name) if err != nil { From 0b6b167c96224af3f5041fa437ae77650735b172 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 21 Aug 2026 22:38:29 +0000 Subject: [PATCH 18/49] fix: use response metadata to terminate api-key list pagination The six paginated list methods in this connector all generate page+1 unconditionally when the current page returned any items, so the last page is always fetched (and the loop never terminates on the API's own signal). This commit adds an explicit page size and a hasMoreAPIKeyPages helper that reads TotalFilteredCount from the ListAPIKeys response metadata and compares page*size+count against it. Co-authored-by: c1-squire-dev[bot] --- pkg/connector/api_token.go | 8 +++++--- pkg/connector/helpers.go | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) diff --git a/pkg/connector/api_token.go b/pkg/connector/api_token.go index 26a2ef72..a38c3440 100644 --- a/pkg/connector/api_token.go +++ b/pkg/connector/api_token.go @@ -18,6 +18,8 @@ import ( "google.golang.org/grpc/status" ) +const defaultV2PageSize = 100 + type apiTokenBuilder struct { resourceType *v2.ResourceType wrapper *client.DatadogClient @@ -86,7 +88,7 @@ func (o *apiTokenBuilder) List( return nil, nil, err } - res, err := o.wrapper.ListAPIKeys(ctx, datadogV2.NewListAPIKeysOptionalParameters().WithPageNumber(page)) + res, err := o.wrapper.ListAPIKeys(ctx, datadogV2.NewListAPIKeysOptionalParameters().WithPageNumber(page).WithPageSize(defaultV2PageSize)) if err != nil { return nil, nil, fmt.Errorf("error listing api tokens: %w", err) } @@ -146,10 +148,10 @@ func (o *apiTokenBuilder) List( return nil, nil, err } ret = append(ret, rv) - } +} nextPageToken := "" - if len(apiTokens) != 0 { + if hasMoreAPIKeyPages(res, page, int64(len(apiTokens)), defaultV2PageSize) { nextPageToken, err = getPageTokenFromPage(bag, page+1) if err != nil { return nil, nil, fmt.Errorf("baton-datadog: failed to get token from page: %w", err) diff --git a/pkg/connector/helpers.go b/pkg/connector/helpers.go index 890b63d2..ba47dff7 100644 --- a/pkg/connector/helpers.go +++ b/pkg/connector/helpers.go @@ -5,6 +5,7 @@ import ( "strconv" "strings" + "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/pagination" ) @@ -109,3 +110,19 @@ func getPageTokenFromPage(bag *pagination.Bag, page int64) (string, error) { return pageToken, nil } + +// hasMoreAPIKeyPages reports whether a ListAPIKeys response has additional pages. +func hasMoreAPIKeyPages(res *datadogV2.APIKeysResponse, page int64, count int64, pageSize int64) bool { + if res == nil { + return count != 0 + } + meta, ok := res.GetMetaOk() + if !ok || meta == nil { + return count != 0 + } + pageMeta := meta.GetPage() + total := pageMeta.GetTotalFilteredCount() + return page*pageSize+count < total +} + + From 454f1194b1f79391483e02710c05a938f2134375 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 21 Aug 2026 22:38:38 +0000 Subject: [PATCH 19/49] fix: page through all results in FindAPIKeyByName; test cross-page exact match FindAPIKeyByName only read the first page of filtered results, so an exact name match could land on page 2+ in an org with more than 100 keys matching the filter. Loop until an exact match is found or the pages are exhausted. Add a subtest that returns the match on page 1 (0-indexed). Co-authored-by: c1-squire-dev[bot] --- pkg/client/client.go | 28 +++++++++++++++++----------- pkg/client/client_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 11 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index dc022f46..cb4f6742 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -201,19 +201,25 @@ func (w *DatadogClient) CreateAPIKey(ctx context.Context, name string) (*IssuedA func (w *DatadogClient) FindAPIKeyByName(ctx context.Context, name string) (*datadogV2.PartialAPIKey, error) { ctx = w.withAuthContext(ctx) api := datadogV2.NewKeyManagementApi(w.officialClient) - response, httpRes, err := api.ListAPIKeys(ctx, *datadogV2.NewListAPIKeysOptionalParameters().WithFilter(name).WithPageSize(100)) - if httpRes != nil { - defer httpRes.Body.Close() - } - if err != nil { - return nil, wrapOfficialClientError("find API key by name", httpRes, err) - } - for _, key := range response.GetData() { - if key.Attributes != nil && key.Attributes.GetName() == name { - return &key, nil + const pageSize = int64(100) + for page := int64(0); ; page++ { + params := *datadogV2.NewListAPIKeysOptionalParameters().WithFilter(name).WithPageSize(pageSize).WithPageNumber(page) + response, httpRes, err := api.ListAPIKeys(ctx, params) + if httpRes != nil { + httpRes.Body.Close() + } + if err != nil { + return nil, wrapOfficialClientError("find API key by name", httpRes, err) + } + for _, key := range response.GetData() { + if key.Attributes != nil && key.Attributes.GetName() == name { + return &key, nil + } + } + if int64(len(response.GetData())) < pageSize { + return nil, nil } } - return nil, nil } func (w *DatadogClient) GetAPIKey(ctx context.Context, id string) (*datadogV2.APIKeyResponse, error) { diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index d67261a0..f0ba88e8 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -90,6 +90,34 @@ func TestAPIKeyManagement(t *testing.T) { } }) + t.Run("find by name returns the exact match from a later page", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assertEqual(t, http.MethodGet, r.Method, "HTTP method should match") + q := r.URL.Query() + page := q.Get("page[number]") + w.Header().Set("Content-Type", "application/json") + if page == "" || page == "0" { + // Page 0: 100 filler keys, none matching. + entries := make([]string, 100) + for i := range entries { + entries[i] = fmt.Sprintf(`{"id":"key-%d","type":"api_keys","attributes":{"name":"c1-other-%d"}}`, i, i) + } + _, _ = fmt.Fprintf(w, `{"data":[%s],"meta":{"page":{"total_filtered_count":101}}}`, strings.Join(entries, ",")) + return + } + // Page 1: one exact match. + _, _ = w.Write([]byte(`{"data":[{"id":"key-match","type":"api_keys","attributes":{"name":"c1-request"}}],"meta":{"page":{"total_filtered_count":101}}}`)) + })) + defer server.Close() + + found, err := newOfficialTestClient(server.URL).FindAPIKeyByName(context.Background(), "c1-request") + assertNoError(t, err, "find API key by name should succeed") + assertNotNil(t, found, "expected an exact match across pages") + if found != nil { + assertEqual(t, "key-match", found.GetId(), "should find the key on page 1, not page 0") + } + }) + t.Run("delete maps a provider 404 to not found", func(t *testing.T) { server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { assertEqual(t, http.MethodDelete, r.Method, "HTTP method should match") From 1c189105a90c3bafd198a5e5e19e564054d85ca8 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:06:39 +0000 Subject: [PATCH 20/49] fix: retarget credential issuance at Datadog service-account application keys SPEC-07 (ductone/pqprime, JUDGED) rules that organization API keys and current-user application keys are not an honest credential-issuance mapping: neither has a reliable non-human owner. Issue now mints a Datadog service-account application key instead, live-rechecking ACCOUNT_TYPE_SERVICE via GetUser before minting, so credential ownership actually reflects who holds the key. Adds a distinct "service-account-application-key" resource type so synced secrets carry the credential kind as a structured signal (resource type id plus SecretTrait.credential_detail), not display-name prose; organization API keys keep syncing and stay deletable under the existing "api-key" type, unchanged. Delete threads the owning service account through ResourceDeleterV2Limited.Delete's existing parentResourceID parameter rather than inventing a packed handle string, so the interface's shape (two ResourceId parameters in, secret never one of them) is unchanged from what apiTokenBuilder already had. Co-authored-by: c1-squire-dev[bot] --- baton_capabilities.json | 41 ++- docs/connector.mdx | 13 +- pkg/client/client.go | 99 +++++ pkg/client/client_test.go | 111 ++++++ pkg/connector/application_key.go | 202 ++++++++++ pkg/connector/connector.go | 2 +- pkg/connector/credential_lifecycle_test.go | 406 ++++++++++++++++----- pkg/connector/resource_types.go | 28 +- pkg/connector/users.go | 49 ++- 9 files changed, 834 insertions(+), 117 deletions(-) create mode 100644 pkg/connector/application_key.go diff --git a/baton_capabilities.json b/baton_capabilities.json index 992d5da0..21a0877b 100644 --- a/baton_capabilities.json +++ b/baton_capabilities.json @@ -4,7 +4,7 @@ { "resourceType": { "id": "api-key", - "displayName": "API Key", + "displayName": "Organization API Key", "traits": [ "TRAIT_SECRET" ], @@ -27,7 +27,7 @@ ] } ], - "description": "Credential issuance creates keys owned by the connector's Datadog principal, not the selected Datadog user." + "description": "A Datadog organization API key. Owned by the org, not by any single Datadog identity; not used for credential issuance by this connector." }, "capabilities": [ "CAPABILITY_SYNC", @@ -106,6 +106,40 @@ ] } }, + { + "resourceType": { + "id": "service-account-application-key", + "displayName": "Service Account Application Key", + "traits": [ + "TRAIT_SECRET" + ], + "annotations": [ + { + "@type": "type.googleapis.com/c1.connector.v2.SkipEntitlementsAndGrants" + }, + { + "@type": "type.googleapis.com/c1.connector.v2.CapabilityPermissions", + "permissions": [ + { + "permission": "user_access_manage" + } + ] + } + ], + "description": "A Datadog application key owned by one service-account identity. Distinct from an org API key (\"api-key\"): scoped to, and deleted through, its owning service account." + }, + "capabilities": [ + "CAPABILITY_SYNC", + "CAPABILITY_RESOURCE_DELETE" + ], + "permissions": { + "permissions": [ + { + "permission": "user_access_manage" + } + ] + } + }, { "resourceType": { "id": "team", @@ -179,8 +213,9 @@ "options": [ { "option": "CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY", + "customScopesAllowed": true, "resourceMode": "CREDENTIAL_RESOURCE_MODE_DISCOVERABLE", - "secretResourceTypeId": "api-key" + "secretResourceTypeId": "service-account-application-key" } ], "preferredOption": "CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY" diff --git a/docs/connector.mdx b/docs/connector.mdx index 8356cf38..980e95b6 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -18,17 +18,18 @@ sidebarTitle: "Datadog" | Roles | | | | | | Teams | | | | | | Schedules | * | | | | -| Secrets - API keys | | | | | +| Secrets - Organization API keys | | | | | +| Secrets - Service account application keys | | | | | -[This connector can sync secrets](/product/admin/inventory) and display them on the **Inventory** page. +[This connector can sync secrets](/product/admin/inventory) and display them on the **Inventory** page. Organization API keys and service account application keys are synced and shown as distinct secret kinds; only application keys owned by a Datadog service account can be issued. -API keys can be issued and revoked through C1 when **Sync secrets** is enabled. Datadog does not support an expiration date when creating an API key. +An application key can be issued and revoked through C1 when **Sync secrets** is enabled, provided the selected Datadog user is a service account. Datadog does not support an expiration date when creating an application key. -Datadog creates issued API keys under the connector's authenticated Datadog user or service account. The selected Datadog user is recorded in C1 for tracking, but does not become the provider-side API-key owner or limit the key's provider-side scope. +Credential issuance targets a Datadog service account only. C1 re-checks at issuance time that the selected user is still a service account, and refuses to issue against a human user. The issued application key is owned by, and scoped to, that service account. -*Schedules and API-key issuance are not enabled by default. Enable **Sync schedules** or **Sync secrets**, respectively, when configuring the connector. +*Schedules and application-key issuance are not enabled by default. Enable **Sync schedules** or **Sync secrets**, respectively, when configuring the connector. ### Connector actions @@ -47,7 +48,7 @@ Configuring the connector requires you to pass in credentials generated in Datad A user with the **Connector Administrator** or **Super Administrator** role in C1 and the **Datadog Admin** or **Datadog standard** role in Datadog must perform this task. -If your user has a custom Datadog role, make sure it includes **User App Keys**, **User Access Invite**, and **User Access Manage** to create, update, enable, and disable users from C1. If you enable **Sync secrets** to issue or revoke API keys, also add **API Keys Read**, **API Keys Write**, and **API Keys Delete**. +If your user has a custom Datadog role, make sure it includes **User App Keys**, **User Access Invite**, and **User Access Manage** to create, update, enable, and disable users from C1. **User Access Manage** also governs issuing, syncing, and revoking service account application keys. If you enable **Sync secrets** to sync or revoke organization API keys, also add **API Keys Read**, **API Keys Write**, and **API Keys Delete**. ### Locate your Datadog site diff --git a/pkg/client/client.go b/pkg/client/client.go index cb4f6742..156fd297 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -270,6 +270,105 @@ func (w *DatadogClient) DeleteAPIKey(ctx context.Context, id string) error { return nil } +// IssuedApplicationKey is the result of issuing a Datadog service-account +// application key: the provider handle (ID), the one-time plaintext secret, +// and the service-account id that owns it. ServiceAccountID is required at +// delete time -- DeleteServiceAccountApplicationKey has no lookup-by-id-alone +// form -- so callers must retain it (see application_key.go's Delete doc +// comment: it travels via ResourceDeleterV2Limited.Delete's parentResourceID +// parameter, not a packed handle string). +type IssuedApplicationKey struct { + ID string + Secret string + ServiceAccountID string +} + +// CreateServiceAccountApplicationKey issues a new application key scoped to +// and owned by the given Datadog service account. Scopes may be empty (an +// unscoped application key). +func (w *DatadogClient) CreateServiceAccountApplicationKey(ctx context.Context, serviceAccountID, name string, scopes []string) (*IssuedApplicationKey, error) { + ctx = w.withAuthContext(ctx) + api := datadogV2.NewServiceAccountsApi(w.officialClient) + attrs := *datadogV2.NewApplicationKeyCreateAttributes(name) + if len(scopes) > 0 { + attrs.SetScopes(scopes) + } + data := *datadogV2.NewApplicationKeyCreateData(attrs, datadogV2.APPLICATIONKEYSTYPE_APPLICATION_KEYS) + response, httpRes, err := api.CreateServiceAccountApplicationKey(ctx, serviceAccountID, *datadogV2.NewApplicationKeyCreateRequest(data)) + if httpRes != nil { + defer httpRes.Body.Close() + } + if err != nil { + return nil, wrapOfficialClientError("create service account application key", httpRes, err) + } + appKey := response.GetData() + if appKey.Id == nil || appKey.Attributes == nil || appKey.Attributes.Key == nil || *appKey.Attributes.Key == "" { + return nil, fmt.Errorf("create service account application key response omitted id or key") + } + return &IssuedApplicationKey{ID: *appKey.Id, Secret: *appKey.Attributes.Key, ServiceAccountID: serviceAccountID}, nil +} + +// FindServiceAccountApplicationKeyByName returns an exact name match among a +// single service account's application keys, if one exists. Mirrors +// FindAPIKeyByName's exact-match-after-filter, paginated pattern, scoped to +// one service account instead of the whole org. +func (w *DatadogClient) FindServiceAccountApplicationKeyByName(ctx context.Context, serviceAccountID, name string) (*datadogV2.PartialApplicationKey, error) { + ctx = w.withAuthContext(ctx) + api := datadogV2.NewServiceAccountsApi(w.officialClient) + const pageSize = int64(100) + for page := int64(0); ; page++ { + params := *datadogV2.NewListServiceAccountApplicationKeysOptionalParameters().WithFilter(name).WithPageSize(pageSize).WithPageNumber(page) + response, httpRes, err := api.ListServiceAccountApplicationKeys(ctx, serviceAccountID, params) + if httpRes != nil { + httpRes.Body.Close() + } + if err != nil { + return nil, wrapOfficialClientError("find service account application key by name", httpRes, err) + } + for _, key := range response.GetData() { + if key.Attributes != nil && key.Attributes.GetName() == name { + return &key, nil + } + } + if int64(len(response.GetData())) < pageSize { + return nil, nil + } + } +} + +// ListServiceAccountApplicationKeys lists every application key owned by the +// given service account, one page at a time via pageNumber/pageSize. +func (w *DatadogClient) ListServiceAccountApplicationKeys(ctx context.Context, serviceAccountID string, pageNumber, pageSize int64) (*datadogV2.ListApplicationKeysResponse, error) { + ctx = w.withAuthContext(ctx) + api := datadogV2.NewServiceAccountsApi(w.officialClient) + params := *datadogV2.NewListServiceAccountApplicationKeysOptionalParameters().WithPageSize(pageSize).WithPageNumber(pageNumber) + resp, httpRes, err := api.ListServiceAccountApplicationKeys(ctx, serviceAccountID, params) + if httpRes != nil { + defer httpRes.Body.Close() + } + if err != nil { + return nil, wrapOfficialClientError("list service account application keys", httpRes, err) + } + return &resp, nil +} + +// DeleteServiceAccountApplicationKey deletes an application key owned by the +// given service account. Unlike DeleteAPIKey, Datadog's API requires both the +// owning service-account id and the key id -- there is no delete-by-key-id-alone +// form for this credential type. +func (w *DatadogClient) DeleteServiceAccountApplicationKey(ctx context.Context, serviceAccountID, appKeyID string) error { + ctx = w.withAuthContext(ctx) + api := datadogV2.NewServiceAccountsApi(w.officialClient) + httpRes, err := api.DeleteServiceAccountApplicationKey(ctx, serviceAccountID, appKeyID) + if httpRes != nil { + defer httpRes.Body.Close() + } + if err != nil { + return wrapOfficialClientError("delete service account application key", httpRes, err) + } + return nil +} + // Wrapper methods that handle HTTP response body closing automatically // ListRoleUsers lists users for a specific role and automatically handles HTTP response body closing. diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index f0ba88e8..792d0260 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -134,6 +134,117 @@ func TestAPIKeyManagement(t *testing.T) { }) } +func TestServiceAccountApplicationKeyManagement(t *testing.T) { + const serviceAccountID = "sa-1" + appKeysPath := "/api/v2/service_accounts/" + serviceAccountID + "/application_keys" + + t.Run("create returns issued material and the owning service account id", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assertEqual(t, http.MethodPost, r.Method, "HTTP method should match") + assertEqual(t, appKeysPath, r.URL.Path, "request path should match") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":{"id":"appkey-id","type":"application_keys","attributes":{"key":"plaintext-app-key","name":"c1-request"}}}`)) + })) + defer server.Close() + + issued, err := newOfficialTestClient(server.URL).CreateServiceAccountApplicationKey(context.Background(), serviceAccountID, "c1-request", nil) + assertNoError(t, err, "create service account application key should succeed") + assertEqual(t, "appkey-id", issued.ID, "issued application key ID should match") + assertEqual(t, "plaintext-app-key", issued.Secret, "issued application key material should match") + assertEqual(t, serviceAccountID, issued.ServiceAccountID, "issued application key should record its owning service account") + }) + + t.Run("create sends requested scopes", func(t *testing.T) { + var body string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + buf := make([]byte, r.ContentLength) + _, _ = r.Body.Read(buf) + body = string(buf) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":{"id":"appkey-id","type":"application_keys","attributes":{"key":"plaintext-app-key","name":"c1-request"}}}`)) + })) + defer server.Close() + + _, err := newOfficialTestClient(server.URL).CreateServiceAccountApplicationKey(context.Background(), serviceAccountID, "c1-request", []string{"dashboards_read", "metrics_read"}) + assertNoError(t, err, "create service account application key should succeed") + assertContains(t, body, "dashboards_read", "request body should include the requested scopes") + assertContains(t, body, "metrics_read", "request body should include the requested scopes") + }) + + t.Run("create rejects a response without plaintext material", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":{"id":"appkey-id","type":"application_keys","attributes":{}}}`)) + })) + defer server.Close() + + _, err := newOfficialTestClient(server.URL).CreateServiceAccountApplicationKey(context.Background(), serviceAccountID, "c1-request", nil) + assertError(t, err, "create service account application key should reject missing plaintext material") + }) + + t.Run("find by name returns the exact match, scoped to the service account", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assertEqual(t, http.MethodGet, r.Method, "HTTP method should match") + assertEqual(t, appKeysPath, r.URL.Path, "request path should match") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[ + {"id":"appkey-partial","type":"application_keys","attributes":{"name":"c1-request-old"}}, + {"id":"appkey-exact","type":"application_keys","attributes":{"name":"c1-request"}} + ]}`)) + })) + defer server.Close() + + found, err := newOfficialTestClient(server.URL).FindServiceAccountApplicationKeyByName(context.Background(), serviceAccountID, "c1-request") + assertNoError(t, err, "find application key by name should succeed") + assertNotNil(t, found, "expected an exact match") + assertEqual(t, "appkey-exact", found.GetId(), "exact match should be the id whose name matches exactly") + }) + + t.Run("find by name ignores a non-exact partial match", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[{"id":"appkey-partial","type":"application_keys","attributes":{"name":"c1-request-old"}}]}`)) + })) + defer server.Close() + + found, err := newOfficialTestClient(server.URL).FindServiceAccountApplicationKeyByName(context.Background(), serviceAccountID, "c1-request") + assertNoError(t, err, "find application key by name should succeed even with no exact match") + if found != nil { + t.Fatalf("expected no exact match, got %+v", found) + } + }) + + t.Run("list pages through all application keys for the service account", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assertEqual(t, http.MethodGet, r.Method, "HTTP method should match") + assertEqual(t, appKeysPath, r.URL.Path, "request path should match") + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":[{"id":"appkey-1","type":"application_keys","attributes":{"name":"c1-req-1"}}]}`)) + })) + defer server.Close() + + resp, err := newOfficialTestClient(server.URL).ListServiceAccountApplicationKeys(context.Background(), serviceAccountID, 0, 100) + assertNoError(t, err, "list service account application keys should succeed") + assertEqualInt(t, 1, len(resp.GetData()), "expected one application key") + }) + + t.Run("delete maps a provider 404 to not found", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assertEqual(t, http.MethodDelete, r.Method, "HTTP method should match") + assertEqual(t, appKeysPath+"/appkey-id", r.URL.Path, "request path should include both the service account and application key ids") + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":["Not found"]}`)) + })) + defer server.Close() + + err := newOfficialTestClient(server.URL).DeleteServiceAccountApplicationKey(context.Background(), serviceAccountID, "appkey-id") + if status.Code(err) != codes.NotFound { + t.Fatalf("DeleteServiceAccountApplicationKey() error code = %s, want %s; error = %v", status.Code(err), codes.NotFound, err) + } + }) +} + // Helper function to check if a value is not nil. func assertNotNil(t *testing.T, value interface{}, message string) { t.Helper() diff --git a/pkg/connector/application_key.go b/pkg/connector/application_key.go new file mode 100644 index 00000000..915b261f --- /dev/null +++ b/pkg/connector/application_key.go @@ -0,0 +1,202 @@ +package connector + +import ( + "context" + "fmt" + "time" + + "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" + "github.com/conductorone/baton-datadog/pkg/client" + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/conductorone/baton-sdk/pkg/types/resource" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +type applicationKeyBuilder struct { + resourceType *v2.ResourceType + wrapper *client.DatadogClient +} + +var _ connectorbuilder.ResourceSyncerV2 = &applicationKeyBuilder{} +var _ connectorbuilder.ResourceDeleterV2Limited = &applicationKeyBuilder{} + +func newApplicationKeyBuilder(wrapper *client.DatadogClient) *applicationKeyBuilder { + return &applicationKeyBuilder{ + resourceType: serviceAccountApplicationKeyResourceType, + wrapper: wrapper, + } +} + +func (o *applicationKeyBuilder) ResourceType(_ context.Context) *v2.ResourceType { + return o.resourceType +} + +func (o *applicationKeyBuilder) Entitlements(_ context.Context, _ *v2.Resource, _ resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) { + return nil, nil, nil +} + +func (o *applicationKeyBuilder) Grants(_ context.Context, _ *v2.Resource, _ resource.SyncOpAttrs) ([]*v2.Grant, *resource.SyncOpResults, error) { + return nil, nil, nil +} + +// Delete removes a service-account application key. Datadog's +// DeleteServiceAccountApplicationKey has no delete-by-key-id-alone form: it +// requires both the owning service-account id and the application-key id. +// This connector carries the service-account id through +// ResourceDeleterV2Limited.Delete's parentResourceID parameter -- the SDK's +// own typed slot for exactly this -- rather than packing both ids into a +// single opaque handle string. The handle (resourceID) is still the bare +// provider application-key id, the same shape apiTokenBuilder uses for +// organization API keys. +// +// This is a deliberate design choice, not an accident: no C1 caller today +// constructs a DeleteResourceRequest for credential deletion with either +// parentResourceID populated or a packed composite handle -- both shapes are +// equally unexercised by any real caller as of this writing (verified +// against ductone/c1's source). Choosing parentResourceID keeps the handle +// format generic across credential types (a bare provider id, like every +// other secret this connector syncs) instead of inventing a Datadog-specific +// packing convention, which matters once other connectors need the same +// "delete needs two ids" shape (e.g. GCP service-account keys). The +// ResourceDeleterV2Limited.Delete signature itself is unchanged by this +// choice -- it already took two *v2.ResourceId parameters; apiTokenBuilder +// (organization API keys, which need only one id) simply discards the +// second one. Whatever C1-side caller eventually populates parentResourceID +// for a real delete is a platform-level change this file does not make and +// does not depend on to be correct: until that caller exists, Delete fails +// closed with InvalidArgument rather than guessing. +func (o *applicationKeyBuilder) Delete(ctx context.Context, resourceID *v2.ResourceId, parentResourceID *v2.ResourceId) (annotations.Annotations, error) { + if resourceID == nil { + return nil, status.Error(codes.InvalidArgument, "baton-datadog: service account application key id is required") + } + appKeyID := resourceID.GetResource() + if isMalformedAPIKeyHandle(appKeyID) { + return nil, status.Errorf(codes.InvalidArgument, "baton-datadog: service account application key id %q is malformed", appKeyID) + } + if parentResourceID == nil || parentResourceID.GetResourceType() != userResourceType.Id { + return nil, status.Error(codes.InvalidArgument, "baton-datadog: the owning service account id is required to delete a service account application key") + } + serviceAccountID := parentResourceID.GetResource() + if isMalformedAPIKeyHandle(serviceAccountID) { + return nil, status.Errorf(codes.InvalidArgument, "baton-datadog: owning service account id %q is malformed", serviceAccountID) + } + if err := o.wrapper.DeleteServiceAccountApplicationKey(ctx, serviceAccountID, appKeyID); err != nil { + if status.Code(err) == codes.NotFound { + return nil, nil + } + return nil, fmt.Errorf("baton-datadog: delete service account application key: %w", err) + } + return nil, nil +} + +// List syncs every application key owned by a Datadog service account. It +// pages through users (the same page cursor shape apiTokenBuilder/userBuilder +// use), and for each service account found in a page, fully drains that +// service account's own application-key pages via the dedicated +// service-account-scoped list API SPEC-07 requires -- not the org-wide +// application-key list, which would include human-owned keys this +// connector's issuance mapping deliberately never targets. +func (o *applicationKeyBuilder) List( + ctx context.Context, + _ *v2.ResourceId, + opts resource.SyncOpAttrs, +) ([]*v2.Resource, *resource.SyncOpResults, error) { + bag, page, err := parsePageToken(opts.PageToken.Token, &v2.ResourceId{ResourceType: o.resourceType.Id}) + if err != nil { + return nil, nil, err + } + + users, err := o.wrapper.ListUsers(ctx, datadogV2.NewListUsersOptionalParameters().WithPageNumber(page)) + if err != nil { + return nil, nil, fmt.Errorf("baton-datadog: list users while syncing service account application keys: %w", err) + } + + var ret []*v2.Resource + for _, user := range users.GetData() { + if user.Attributes == nil || !user.Attributes.GetServiceAccount() { + continue + } + serviceAccountID := user.GetId() + if serviceAccountID == "" { + continue + } + serviceAccountResourceID := &v2.ResourceId{ResourceType: userResourceType.Id, Resource: serviceAccountID} + + for appKeyPage := int64(0); ; appKeyPage++ { + resp, err := o.wrapper.ListServiceAccountApplicationKeys(ctx, serviceAccountID, appKeyPage, defaultV2PageSize) + if err != nil { + return nil, nil, fmt.Errorf("baton-datadog: list application keys for service account %q: %w", serviceAccountID, err) + } + keys := resp.GetData() + for _, key := range keys { + if key.Id == nil { + continue + } + rv, err := applicationKeyResource(*key.Id, serviceAccountResourceID, key.Attributes) + if err != nil { + return nil, nil, err + } + ret = append(ret, rv) + } + if int64(len(keys)) < defaultV2PageSize { + break + } + } + } + + nextPageToken := "" + if len(users.GetData()) != 0 { + nextPageToken, err = getPageTokenFromPage(bag, page+1) + if err != nil { + return nil, nil, fmt.Errorf("baton-datadog: failed to get token from page: %w", err) + } + } + + return ret, &resource.SyncOpResults{NextPageToken: nextPageToken}, nil +} + +// applicationKeyResource builds the synced resource for one service-account +// application key. The type is unambiguous through two structured signals a +// reader (or a future requester-selection surface) can consume without +// inferring anything from prose: the resource type id/display name +// (serviceAccountApplicationKeyResourceType, distinct from apiTokenResourceType), +// and SecretTrait.credential_detail ("datadog.service_account_application_key", +// set via WithSecretDetail below) -- a structured field on the trait, not +// display-name text. WithParentResourceID records the owning service account +// as the resource's parent; see Delete's doc comment for why that is the +// field this connector's delete path relies on. +func applicationKeyResource(appKeyID string, serviceAccountResourceID *v2.ResourceId, attrs *datadogV2.PartialApplicationKeyAttributes) (*v2.Resource, error) { + name := appKeyID + if attrs != nil && attrs.Name != nil { + name = *attrs.Name + } + + options := []resource.SecretTraitOption{ + resource.WithSecretType(v2.SecretTrait_CREDENTIAL_TYPE_STATIC_SECRET), + resource.WithSecretDetail("datadog.service_account_application_key"), + resource.WithSecretCreatedByID(serviceAccountResourceID), + resource.WithSecretIdentityID(serviceAccountResourceID), + } + + resourceOptions := []resource.ResourceOption{ + resource.WithParentResourceID(serviceAccountResourceID), + } + if attrs != nil && attrs.CreatedAt != nil { + createdAt, err := time.Parse(time.RFC3339Nano, *attrs.CreatedAt) + if err != nil { + return nil, fmt.Errorf("baton-datadog: parse application key created_at: %w", err) + } + resourceOptions = append(resourceOptions, resource.WithResourceCreatedAt(createdAt)) + } + + return resource.NewSecretResource( + name, + serviceAccountApplicationKeyResourceType, + appKeyID, + options, + resourceOptions..., + ) +} diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index aef6dee6..5897e84b 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -45,7 +45,7 @@ func (d *Datadog) ResourceSyncers(ctx context.Context) []connectorbuilder.Resour } if d.SyncSecrets { - resourceSyncers = append(resourceSyncers, newApiTokenBuilder(d.wrapper)) + resourceSyncers = append(resourceSyncers, newApiTokenBuilder(d.wrapper), newApplicationKeyBuilder(d.wrapper)) } if d.SyncSchedules { diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index 61e69c9e..73520ac3 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -20,10 +20,9 @@ import ( "google.golang.org/grpc/status" ) -// newCredentialLifecycleServer stands in for Datadog across an Issue+Delete -// round trip. It answers FindAPIKeyByName (GET, empty match), CreateAPIKey -// (POST, returns handle+secret), and DeleteAPIKey (DELETE by handle). Every -// request the connector actually sends is recorded so tests can assert on it. +// recordedRequest captures one request the connector sent to the fake +// Datadog provider, so tests can assert on exactly what left the connector +// (path, query, headers, body) without trusting a success return alone. type recordedRequest struct { method string path string @@ -32,27 +31,46 @@ type recordedRequest struct { body string } -func newCredentialLifecycleServer(t *testing.T, handle, secret, name string) (*httptest.Server, *[]recordedRequest) { +func newLifecycleTestWrapper(serverURL string) *client.DatadogClient { + cfg := datadog.NewConfiguration() + cfg.Servers = datadog.ServerConfigurations{{URL: serverURL}} + return client.NewDatadogClient(nil, datadog.NewAPIClient(cfg), "example.com", "connector-api-key", "connector-app-key") +} + +func recordRequest(t *testing.T, requests *[]recordedRequest, r *http.Request) recordedRequest { + t.Helper() + bodyBytes := make([]byte, 0) + if r.Body != nil { + buf := new(bytes.Buffer) + _, _ = buf.ReadFrom(r.Body) + bodyBytes = buf.Bytes() + } + rec := recordedRequest{ + method: r.Method, + path: r.URL.Path, + query: r.URL.RawQuery, + header: r.Header.Clone(), + body: string(bodyBytes), + } + *requests = append(*requests, rec) + return rec +} + +// --- organization API key (apiTokenBuilder) coverage ----------------------- +// +// apiTokenBuilder / the "api-key" resource type is unchanged by the SPEC-07 +// rework: it still syncs and deletes organization API keys exactly as +// before. It is no longer wired to Issue (see the service-account +// application key coverage below), so these tests seed a fixture directly +// via the client wrapper's CreateAPIKey instead of going through Issue. + +func newOrgAPIKeyServer(t *testing.T, handle, secret, name string) (*httptest.Server, *[]recordedRequest) { t.Helper() requests := &[]recordedRequest{} server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - bodyBytes := make([]byte, 0) - if r.Body != nil { - buf := new(bytes.Buffer) - _, _ = buf.ReadFrom(r.Body) - bodyBytes = buf.Bytes() - } - *requests = append(*requests, recordedRequest{ - method: r.Method, - path: r.URL.Path, - query: r.URL.RawQuery, - header: r.Header.Clone(), - body: string(bodyBytes), - }) + recordRequest(t, requests, r) w.Header().Set("Content-Type", "application/json") switch { - case r.Method == http.MethodGet && r.URL.Path == "/api/v2/api_keys": - _, _ = w.Write([]byte(`{"data":[]}`)) case r.Method == http.MethodPost && r.URL.Path == "/api/v2/api_keys": _, _ = w.Write([]byte(`{"data":{"id":"` + handle + `","type":"api_keys","attributes":{"key":"` + secret + `","name":"` + name + `"}}}`)) case r.Method == http.MethodDelete && r.URL.Path == "/api/v2/api_keys/"+handle: @@ -65,45 +83,29 @@ func newCredentialLifecycleServer(t *testing.T, handle, secret, name string) (*h return server, requests } -func newLifecycleTestWrapper(serverURL string) *client.DatadogClient { - cfg := datadog.NewConfiguration() - cfg.Servers = datadog.ServerConfigurations{{URL: serverURL}} - return client.NewDatadogClient(nil, datadog.NewAPIClient(cfg), "example.com", "connector-api-key", "connector-app-key") -} - -func issueTestCredential(t *testing.T, ctx context.Context, wrapper *client.DatadogClient) *connectorbuilder.CredentialIssueOutput { - t.Helper() - issuer := newCredentialUserBuilder(wrapper) - out, err := issuer.Issue(ctx, &connectorbuilder.CredentialIssueInput{ - IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: "user-1"}, - RequestID: "req-1", - }) - require.NoError(t, err) - return out -} - -// (a) Connector-level delete-by-handle: apiTokenBuilder.Delete must issue the -// provider DELETE for the resource handle, and the plaintext secret must -// never appear anywhere in that request (path, query, headers, or body). +// TestApiTokenBuilderDeleteUsesHandleNotSecret: apiTokenBuilder.Delete must +// issue the provider DELETE for the resource handle, and the plaintext +// secret must never appear anywhere in that request (path, query, headers, +// or body). func TestApiTokenBuilderDeleteUsesHandleNotSecret(t *testing.T) { const ( handle = "handle-abc123" secret = "super-secret-plaintext-value" name = "c1-req-1" ) - server, requests := newCredentialLifecycleServer(t, handle, secret, name) + server, requests := newOrgAPIKeyServer(t, handle, secret, name) defer server.Close() wrapper := newLifecycleTestWrapper(server.URL) ctx := context.Background() - // Issue first so the fake provider has a real handle/secret pair on record, - // then delete strictly by handle -- the way the connector actually calls it. - issued := issueTestCredential(t, ctx, wrapper) - require.Equal(t, handle, issued.Secret.GetId().GetResource()) - require.Equal(t, secret, string(issued.PlaintextData[0].GetBytes())) + issued, err := wrapper.CreateAPIKey(ctx, name) + require.NoError(t, err) + require.Equal(t, handle, issued.ID) + require.Equal(t, secret, issued.Secret) deleter := newApiTokenBuilder(wrapper) - _, err := deleter.Delete(ctx, issued.Secret.GetId(), nil) + resourceID := &v2.ResourceId{ResourceType: apiTokenResourceType.Id, Resource: issued.ID} + _, err = deleter.Delete(ctx, resourceID, nil) require.NoError(t, err) var deleteReq *recordedRequest @@ -127,10 +129,11 @@ func TestApiTokenBuilderDeleteUsesHandleNotSecret(t *testing.T) { } } -// (b) Malformed/missing handle must fail closed before any provider request. -// nil ResourceId, an empty ResourceId.Resource, and a non-empty malformed -// handle (whitespace-only or containing a control character) are all -// validated by pkg/connector/api_token.go:29-36 / isMalformedAPIKeyHandle. +// TestApiTokenBuilderDeleteRejectsMissingHandle: malformed/missing handle +// must fail closed before any provider request. nil ResourceId, an empty +// ResourceId.Resource, and a non-empty malformed handle (whitespace-only or +// containing a control character) are all validated by +// pkg/connector/api_token.go / isMalformedAPIKeyHandle. func TestApiTokenBuilderDeleteRejectsMissingHandle(t *testing.T) { tests := []struct { name string @@ -153,24 +156,174 @@ func TestApiTokenBuilderDeleteRejectsMissingHandle(t *testing.T) { deleter := newApiTokenBuilder(wrapper) _, err := deleter.Delete(context.Background(), tt.resourceID, nil) require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) }) } } -// (c) Handle/secret separation regression at the issuer/connector boundary: -// the returned secret resource ID must equal the provider handle, and must -// never equal the plaintext secret bytes. +// --- service-account application key (credentialUserBuilder.Issue / +// applicationKeyBuilder.Delete) coverage ------------------------------------ +// +// This is the SPEC-07 mapping: Issue targets a Datadog service account +// (live-rechecked via GetUser) and mints an application key scoped to it; +// Delete removes that key through the service-account application-key API +// using the bare application-key id as the handle plus the owning service +// account carried via parentResourceID (see application_key.go's Delete doc +// comment for why parentResourceID, not a packed handle string). + +const ( + testServiceAccountID = "sa-1" +) + +// newServiceAccountAppKeyServer stands in for Datadog across an Issue+Delete +// round trip against a service account. It answers GetUser (service account +// check), the find-by-name list (empty match), CreateServiceAccountApplicationKey +// (returns handle+secret), and DeleteServiceAccountApplicationKey (DELETE by +// handle, scoped to the service account in the request path). Every request +// the connector actually sends is recorded. +func newServiceAccountAppKeyServer(t *testing.T, serviceAccountID, handle, secret, name string) (*httptest.Server, *[]recordedRequest) { + t.Helper() + requests := &[]recordedRequest{} + appKeysPath := "/api/v2/service_accounts/" + serviceAccountID + "/application_keys" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + recordRequest(t, requests, r) + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/users/"+serviceAccountID: + _, _ = w.Write([]byte(`{"data":{"id":"` + serviceAccountID + `","type":"users","attributes":{"service_account":true}}}`)) + case r.Method == http.MethodGet && r.URL.Path == appKeysPath: + _, _ = w.Write([]byte(`{"data":[]}`)) + case r.Method == http.MethodPost && r.URL.Path == appKeysPath: + _, _ = w.Write([]byte(`{"data":{"id":"` + handle + `","type":"application_keys","attributes":{"key":"` + secret + `","name":"` + name + `"}}}`)) + case r.Method == http.MethodDelete && r.URL.Path == appKeysPath+"/"+handle: + w.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected provider request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + } + })) + return server, requests +} + +func issueServiceAccountAppKey(t *testing.T, ctx context.Context, wrapper *client.DatadogClient, serviceAccountID, requestID string) *connectorbuilder.CredentialIssueOutput { + t.Helper() + issuer := newCredentialUserBuilder(wrapper) + out, err := issuer.Issue(ctx, &connectorbuilder.CredentialIssueInput{ + IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: serviceAccountID}, + RequestID: requestID, + CredentialOptions: v2.CredentialIssueOptions_builder{ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build()}.Build(), + }) + require.NoError(t, err) + return out +} + +// TestIssueRequiresServiceAccount: Issue must live-recheck (via GetUser) that +// the target Datadog user is a service account, accepting one and rejecting +// a human user with InvalidArgument, without ever calling +// CreateServiceAccountApplicationKey for the rejected case. +func TestIssueRequiresServiceAccount(t *testing.T) { + t.Run("accepts a service account", func(t *testing.T) { + const ( + handle = "appkey-sa-accept" + secret = "test-fixture-value-accept" + name = "c1-req-accept" + ) + server, _ := newServiceAccountAppKeyServer(t, testServiceAccountID, handle, secret, name) + defer server.Close() + wrapper := newLifecycleTestWrapper(server.URL) + + out := issueServiceAccountAppKey(t, context.Background(), wrapper, testServiceAccountID, "req-accept") + require.Equal(t, handle, out.Secret.GetId().GetResource()) + require.Equal(t, testServiceAccountID, out.Secret.GetParentResourceId().GetResource(), "the issued secret must record its owning service account as its parent resource") + }) + + t.Run("rejects a human user", func(t *testing.T) { + const humanUserID = "user-human-1" + appKeysPath := "/api/v2/service_accounts/" + humanUserID + "/application_keys" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/users/"+humanUserID: + _, _ = w.Write([]byte(`{"data":{"id":"` + humanUserID + `","type":"users","attributes":{"service_account":false}}}`)) + case r.Method == http.MethodPost && r.URL.Path == appKeysPath: + t.Errorf("CreateServiceAccountApplicationKey should not be called for a non-service-account target") + w.WriteHeader(http.StatusInternalServerError) + default: + t.Errorf("unexpected provider request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + } + })) + defer server.Close() + wrapper := newLifecycleTestWrapper(server.URL) + + issuer := newCredentialUserBuilder(wrapper) + out, err := issuer.Issue(context.Background(), &connectorbuilder.CredentialIssueInput{ + IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: humanUserID}, + RequestID: "req-reject", + CredentialOptions: v2.CredentialIssueOptions_builder{ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build()}.Build(), + }) + require.Nil(t, out) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + }) +} + +// TestIssueRefusesDuplicateRequest exercises the +// FindServiceAccountApplicationKeyByName exact-match branch as consumed by +// Issue: when the provider already has an application key named for this +// request (scoped to this service account), Issue must refuse with +// AlreadyExists and must never call CreateServiceAccountApplicationKey. +func TestIssueRefusesDuplicateRequest(t *testing.T) { + const ( + requestID = "req-dup-1" + name = "c1-" + requestID + existingID = "appkey-existing-1" + ) + appKeysPath := "/api/v2/service_accounts/" + testServiceAccountID + "/application_keys" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/users/"+testServiceAccountID: + _, _ = w.Write([]byte(`{"data":{"id":"` + testServiceAccountID + `","type":"users","attributes":{"service_account":true}}}`)) + case r.Method == http.MethodGet && r.URL.Path == appKeysPath: + _, _ = w.Write([]byte(`{"data":[{"id":"` + existingID + `","type":"application_keys","attributes":{"name":"` + name + `"}}]}`)) + case r.Method == http.MethodPost && r.URL.Path == appKeysPath: + t.Errorf("CreateServiceAccountApplicationKey should not be called when a key for this request already exists") + w.WriteHeader(http.StatusInternalServerError) + default: + t.Errorf("unexpected provider request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + } + })) + defer server.Close() + wrapper := newLifecycleTestWrapper(server.URL) + + issuer := newCredentialUserBuilder(wrapper) + out, err := issuer.Issue(context.Background(), &connectorbuilder.CredentialIssueInput{ + IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID}, + RequestID: requestID, + CredentialOptions: v2.CredentialIssueOptions_builder{ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build()}.Build(), + }) + require.Nil(t, out) + require.Error(t, err) + require.Equal(t, codes.AlreadyExists, status.Code(err)) +} + +// TestIssueHandleAndSecretAreDistinct: handle/secret separation regression at +// the issuer/connector boundary -- the returned secret resource ID must be +// the bare provider application-key id, and must never equal the plaintext +// secret bytes. func TestIssueHandleAndSecretAreDistinct(t *testing.T) { const ( handle = "handle-distinct-1" secret = "plaintext-distinct-secret" name = "c1-req-1" ) - server, _ := newCredentialLifecycleServer(t, handle, secret, name) + server, _ := newServiceAccountAppKeyServer(t, testServiceAccountID, handle, secret, name) defer server.Close() wrapper := newLifecycleTestWrapper(server.URL) - issued := issueTestCredential(t, context.Background(), wrapper) + issued := issueServiceAccountAppKey(t, context.Background(), wrapper, testServiceAccountID, "req-1") secretResourceID := issued.Secret.GetId().GetResource() plaintext := string(issued.PlaintextData[0].GetBytes()) @@ -178,11 +331,11 @@ func TestIssueHandleAndSecretAreDistinct(t *testing.T) { require.NotEmpty(t, secretResourceID) require.NotEmpty(t, plaintext) require.NotEqual(t, plaintext, secretResourceID, "the secret resource id must not be (or equal) the plaintext secret") - require.Equal(t, handle, secretResourceID, "the secret resource id must equal the provider-issued handle") + require.Equal(t, handle, secretResourceID, "the secret resource id must equal the provider-issued application key id") } -// (d) Secret-log assertion: capture every log record emitted across a full -// Issue + Delete cycle and assert the synthetic plaintext secret never +// TestIssueAndDeleteNeverLogSecret: capture every log record emitted across a +// full Issue + Delete cycle and assert the synthetic plaintext secret never // appears in them. func TestIssueAndDeleteNeverLogSecret(t *testing.T) { const ( @@ -190,7 +343,7 @@ func TestIssueAndDeleteNeverLogSecret(t *testing.T) { secret = "plaintext-should-never-be-logged" name = "c1-req-1" ) - server, _ := newCredentialLifecycleServer(t, handle, secret, name) + server, _ := newServiceAccountAppKeyServer(t, testServiceAccountID, handle, secret, name) defer server.Close() wrapper := newLifecycleTestWrapper(server.URL) @@ -200,11 +353,11 @@ func TestIssueAndDeleteNeverLogSecret(t *testing.T) { logger := zap.New(core) ctx := ctxzap.ToContext(context.Background(), logger) - issued := issueTestCredential(t, ctx, wrapper) + issued := issueServiceAccountAppKey(t, ctx, wrapper, testServiceAccountID, "req-1") require.Equal(t, secret, string(issued.PlaintextData[0].GetBytes())) - deleter := newApiTokenBuilder(wrapper) - _, err := deleter.Delete(ctx, issued.Secret.GetId(), nil) + deleter := newApplicationKeyBuilder(wrapper) + _, err := deleter.Delete(ctx, issued.Secret.GetId(), issued.Secret.GetParentResourceId()) require.NoError(t, err) require.NoError(t, logger.Sync()) @@ -213,38 +366,107 @@ func TestIssueAndDeleteNeverLogSecret(t *testing.T) { } } -// TestIssueRefusesDuplicateRequest exercises the FindAPIKeyByName exact-match -// branch as consumed by Issue: when the provider already has a key named for -// this request, Issue must refuse with AlreadyExists and must never call -// CreateAPIKey (POST /api/v2/api_keys). -func TestIssueRefusesDuplicateRequest(t *testing.T) { +// TestApplicationKeyBuilderDeleteUsesServiceAccountAPI: applicationKeyBuilder.Delete +// must issue the provider DELETE against the service-account-scoped +// application-key path, using the handle (resourceID) and the owning +// service account (parentResourceID, exactly as Issue recorded it on the +// returned secret's ParentResourceId), and the plaintext secret must never +// appear anywhere in that request. +func TestApplicationKeyBuilderDeleteUsesServiceAccountAPI(t *testing.T) { const ( - requestID = "req-dup-1" - name = "c1-" + requestID - existingID = "handle-existing-1" + handle = "handle-sa-delete-1" + secret = "super-secret-app-key-plaintext" + name = "c1-req-1" ) - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("Content-Type", "application/json") - switch { - case r.Method == http.MethodGet && r.URL.Path == "/api/v2/api_keys": - _, _ = w.Write([]byte(`{"data":[{"id":"` + existingID + `","type":"api_keys","attributes":{"name":"` + name + `"}}]}`)) - case r.Method == http.MethodPost && r.URL.Path == "/api/v2/api_keys": - t.Errorf("CreateAPIKey should not be called when a key for this request already exists") - w.WriteHeader(http.StatusInternalServerError) - default: - t.Errorf("unexpected provider request: %s %s", r.Method, r.URL.Path) - w.WriteHeader(http.StatusInternalServerError) - } - })) + server, requests := newServiceAccountAppKeyServer(t, testServiceAccountID, handle, secret, name) defer server.Close() wrapper := newLifecycleTestWrapper(server.URL) + ctx := context.Background() - issuer := newCredentialUserBuilder(wrapper) - out, err := issuer.Issue(context.Background(), &connectorbuilder.CredentialIssueInput{ - IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: "user-1"}, - RequestID: requestID, - }) - require.Nil(t, out) - require.Error(t, err) - require.Equal(t, codes.AlreadyExists, status.Code(err)) + issued := issueServiceAccountAppKey(t, ctx, wrapper, testServiceAccountID, "req-1") + require.Equal(t, secret, string(issued.PlaintextData[0].GetBytes())) + + deleter := newApplicationKeyBuilder(wrapper) + _, err := deleter.Delete(ctx, issued.Secret.GetId(), issued.Secret.GetParentResourceId()) + require.NoError(t, err) + + var deleteReq *recordedRequest + for i := range *requests { + if (*requests)[i].method == http.MethodDelete { + deleteReq = &(*requests)[i] + } + } + require.NotNil(t, deleteReq, "expected a DELETE request to reach the provider") + require.Equal(t, "/api/v2/service_accounts/"+testServiceAccountID+"/application_keys/"+handle, deleteReq.path) + + if strings.Contains(deleteReq.path, secret) || strings.Contains(deleteReq.query, secret) || strings.Contains(deleteReq.body, secret) { + t.Fatalf("delete request leaked the plaintext secret: %+v", deleteReq) + } +} + +// TestApplicationKeyBuilderDeleteRejectsMalformedHandle: nil ResourceId, an +// empty or control-character handle, a missing/wrong-type/malformed +// parentResourceID (the owning service account) must all fail closed before +// any provider request. Datadog's DeleteServiceAccountApplicationKey needs +// both ids; parentResourceID carries the service account id (see +// application_key.go's Delete doc comment for why that parameter, not a +// packed handle string). +func TestApplicationKeyBuilderDeleteRejectsMalformedHandle(t *testing.T) { + validParent := &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID} + tests := []struct { + name string + resourceID *v2.ResourceId + parentResourceID *v2.ResourceId + }{ + { + name: "nil ResourceId", + resourceID: nil, + parentResourceID: validParent, + }, + { + name: "empty ResourceId.Resource", + resourceID: &v2.ResourceId{ResourceType: serviceAccountApplicationKeyResourceType.Id, Resource: ""}, + parentResourceID: validParent, + }, + { + name: "handle with control character", + resourceID: &v2.ResourceId{ResourceType: serviceAccountApplicationKeyResourceType.Id, Resource: "appkey-\n1"}, + parentResourceID: validParent, + }, + { + name: "nil parentResourceID (owning service account missing)", + resourceID: &v2.ResourceId{ResourceType: serviceAccountApplicationKeyResourceType.Id, Resource: "appkey-1"}, + parentResourceID: nil, + }, + { + name: "wrong-type parentResourceID", + resourceID: &v2.ResourceId{ResourceType: serviceAccountApplicationKeyResourceType.Id, Resource: "appkey-1"}, + parentResourceID: &v2.ResourceId{ResourceType: apiTokenResourceType.Id, Resource: testServiceAccountID}, + }, + { + name: "empty parentResourceID.Resource", + resourceID: &v2.ResourceId{ResourceType: serviceAccountApplicationKeyResourceType.Id, Resource: "appkey-1"}, + parentResourceID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: ""}, + }, + { + name: "parentResourceID with control character", + resourceID: &v2.ResourceId{ResourceType: serviceAccountApplicationKeyResourceType.Id, Resource: "appkey-1"}, + parentResourceID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: "sa-\n1"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Errorf("provider should not be contacted for a %s, got %s %s", tt.name, r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + wrapper := newLifecycleTestWrapper(server.URL) + + deleter := newApplicationKeyBuilder(wrapper) + _, err := deleter.Delete(context.Background(), tt.resourceID, tt.parentResourceID) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + }) + } } diff --git a/pkg/connector/resource_types.go b/pkg/connector/resource_types.go index e387e4e9..68c18b2b 100644 --- a/pkg/connector/resource_types.go +++ b/pkg/connector/resource_types.go @@ -43,16 +43,40 @@ var ( capabilityPermissions("user_access_manage"), ), } + // apiTokenResourceType covers organization-scoped API keys (Datadog's + // "API keys", /api/v2/api_keys): org-wide credentials not owned by any + // single Datadog identity. This connector still syncs and can delete + // them, but Issue no longer targets this type -- an org-scoped key + // issued on behalf of a selected user is not an honest mapping of who + // holds it. See serviceAccountApplicationKeyResourceType for the type + // Issue does target. apiTokenResourceType = &v2.ResourceType{ Id: "api-key", - DisplayName: "API Key", - Description: "Credential issuance creates keys owned by the connector's Datadog principal, not the selected Datadog user.", + DisplayName: "Organization API Key", + Description: "A Datadog organization API key. Owned by the org, not by any single Datadog identity; not used for credential issuance by this connector.", Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_SECRET}, Annotations: annotations.New( &v2.SkipEntitlementsAndGrants{}, capabilityPermissions("api_keys_read", "api_keys_write", "api_keys_delete"), ), } + // serviceAccountApplicationKeyResourceType covers application keys owned + // by a Datadog service-account user (/api/v2/service_accounts/{id}/application_keys). + // This is the resource type credential issuance targets: the key is + // scoped to and owned by one service-account identity, so a synced + // resource of this type is distinguishable from an apiTokenResourceType + // (organization API key) by resource type id, display name, and the + // underlying SecretTrait's credential_detail (see application_key.go). + serviceAccountApplicationKeyResourceType = &v2.ResourceType{ + Id: "service-account-application-key", + DisplayName: "Service Account Application Key", + Description: "A Datadog application key owned by one service-account identity. Distinct from an org API key (\"api-key\"): scoped to, and deleted through, its owning service account.", + Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_SECRET}, + Annotations: annotations.New( + &v2.SkipEntitlementsAndGrants{}, + capabilityPermissions("user_access_manage"), + ), + } scheduleResourceType = &v2.ResourceType{ Id: "schedule", DisplayName: "Schedule", diff --git a/pkg/connector/users.go b/pkg/connector/users.go index 1238165f..8c3c7704 100644 --- a/pkg/connector/users.go +++ b/pkg/connector/users.go @@ -34,48 +34,71 @@ func (u *credentialUserBuilder) IssueCapabilityDetails(context.Context) (*v2.Cre Options: []*v2.CredentialIssueOptionDescriptor{v2.CredentialIssueOptionDescriptor_builder{ Option: v2.CapabilityDetailCredentialOption_CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY, ResourceMode: v2.CredentialResourceMode_CREDENTIAL_RESOURCE_MODE_DISCOVERABLE, - SecretResourceTypeId: apiTokenResourceType.Id, + SecretResourceTypeId: serviceAccountApplicationKeyResourceType.Id, + CustomScopesAllowed: true, }.Build()}, PreferredOption: v2.CapabilityDetailCredentialOption_CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY, }.Build(), nil, nil } +// Issue mints a Datadog service-account application key scoped to and owned +// by the target identity. Per SPEC-07 (the judged Datadog credential-issuance +// design), this is the only honest issuance mapping this connector supports: +// an organization API key or a current-user application key has no reliable +// non-human owner, so Issue targets a service-account application key +// instead, gated on a live re-check that the target is actually a Datadog +// service account (its user record may have changed since it was last +// synced). func (u *credentialUserBuilder) Issue(ctx context.Context, input *connectorbuilder.CredentialIssueInput) (*connectorbuilder.CredentialIssueOutput, error) { if input == nil || input.IdentityID == nil || input.IdentityID.GetResourceType() != userResourceType.Id { return nil, status.Error(codes.InvalidArgument, "baton-datadog: a Datadog user identity is required") } + serviceAccountID := input.IdentityID.GetResource() + + userResp, err := u.wrapper.GetUser(ctx, serviceAccountID) + if err != nil { + return nil, fmt.Errorf("baton-datadog: look up Datadog user %q: %w", serviceAccountID, err) + } + if !userResp.GetData().Attributes.GetServiceAccount() { + return nil, status.Errorf(codes.InvalidArgument, "baton-datadog: Datadog user %q is not a service account; credential issuance only targets service accounts", serviceAccountID) + } + name := "c1-" + input.RequestID - existing, err := u.wrapper.FindAPIKeyByName(ctx, name) + existing, err := u.wrapper.FindServiceAccountApplicationKeyByName(ctx, serviceAccountID, name) if err != nil { - return nil, fmt.Errorf("baton-datadog: look up API key for request %q: %w", input.RequestID, err) + return nil, fmt.Errorf("baton-datadog: look up application key for request %q: %w", input.RequestID, err) } if existing != nil { - return nil, status.Errorf(codes.AlreadyExists, "baton-datadog: API key for request %q may already exist; refusing to issue a duplicate", input.RequestID) + return nil, status.Errorf(codes.AlreadyExists, "baton-datadog: application key for request %q may already exist; refusing to issue a duplicate", input.RequestID) } - key, err := u.wrapper.CreateAPIKey(ctx, name) + + scopes := input.CredentialOptions.GetApiKey().GetScopes() + key, err := u.wrapper.CreateServiceAccountApplicationKey(ctx, serviceAccountID, name, scopes) if err != nil { - return nil, fmt.Errorf("baton-datadog: create API key: %w", err) + return nil, fmt.Errorf("baton-datadog: create service account application key: %w", err) } + secretTraitOptions := []rs.SecretTraitOption{ rs.WithSecretCreatedByID(input.IdentityID), rs.WithSecretIdentityID(input.IdentityID), rs.WithSecretType(v2.SecretTrait_CREDENTIAL_TYPE_STATIC_SECRET), - rs.WithSecretDetail("datadog.api_key"), + rs.WithSecretDetail("datadog.service_account_application_key"), } - secret, err := rs.NewSecretResource(name, apiTokenResourceType, key.ID, secretTraitOptions) + secret, err := rs.NewSecretResource(name, serviceAccountApplicationKeyResourceType, key.ID, secretTraitOptions, rs.WithParentResourceID(input.IdentityID)) if err != nil { - if deleteErr := u.wrapper.DeleteAPIKey(ctx, key.ID); deleteErr != nil { - ctxzap.Extract(ctx).Warn("failed to clean up Datadog API key after resource construction error", - zap.String("api_key_id", key.ID), + if deleteErr := u.wrapper.DeleteServiceAccountApplicationKey(ctx, serviceAccountID, key.ID); deleteErr != nil { + ctxzap.Extract(ctx).Warn("failed to clean up Datadog service account application key after resource construction error", + zap.String("service_account_id", serviceAccountID), + zap.String("application_key_id", key.ID), zap.Error(deleteErr), ) } - return nil, fmt.Errorf("baton-datadog: build API key secret resource: %w", err) + return nil, fmt.Errorf("baton-datadog: build service account application key secret resource: %w", err) } return &connectorbuilder.CredentialIssueOutput{ Secret: secret, PlaintextData: []*v2.PlaintextData{ - v2.PlaintextData_builder{Name: "api_key", Bytes: []byte(key.Secret)}.Build(), + v2.PlaintextData_builder{Name: "application_key", Bytes: []byte(key.Secret)}.Build(), }, ResourceMode: v2.CredentialResourceMode_CREDENTIAL_RESOURCE_MODE_DISCOVERABLE, }, nil From 49e87012815c4bea5e844212d1f9430cea721bb9 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:29:12 +0000 Subject: [PATCH 21/49] fix: address pr-review findings on the service-account application key rework Blocking: - retarget the opt-in live smoke test at a real service account and the service-account application-key APIs; it previously called GetUser with a synthetic id (guaranteed 404) and verified/cleaned up through the org API-key endpoints, which would have leaked a live application key on a real run. Suggestions: - map a GetUser 404 during Issue's live service-account recheck to codes.NotFound instead of an unwrapped error. - bound the two new paginated list loops (FindServiceAccountApplicationKeyByName, applicationKeyBuilder.List) so a provider that ignores page[number] fails closed instead of looping forever. - read the scope-assertion test body with io.ReadAll instead of a single Body.Read, which can short-read. - drop the now-unexercised api_keys_write permission from the org API-key resource type (issuance no longer creates org keys) and from the docs paragraph describing it; regenerate capabilities. - document that revoking a service-account application key requires the caller to supply the owning service account, since no caller does that yet. Co-authored-by: c1-squire-dev[bot] --- baton_capabilities.json | 6 -- docs/connector.mdx | 4 +- pkg/client/client.go | 8 +- pkg/client/client_test.go | 4 +- pkg/connector/application_key.go | 12 ++- pkg/connector/credential_smoke_test.go | 108 +++++++++++++++++-------- pkg/connector/resource_types.go | 8 +- pkg/connector/users.go | 3 + 8 files changed, 105 insertions(+), 48 deletions(-) diff --git a/baton_capabilities.json b/baton_capabilities.json index 21a0877b..a6351929 100644 --- a/baton_capabilities.json +++ b/baton_capabilities.json @@ -18,9 +18,6 @@ { "permission": "api_keys_read" }, - { - "permission": "api_keys_write" - }, { "permission": "api_keys_delete" } @@ -38,9 +35,6 @@ { "permission": "api_keys_read" }, - { - "permission": "api_keys_write" - }, { "permission": "api_keys_delete" } diff --git a/docs/connector.mdx b/docs/connector.mdx index 980e95b6..1018dc75 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -27,6 +27,8 @@ An application key can be issued and revoked through C1 when **Sync secrets** is Credential issuance targets a Datadog service account only. C1 re-checks at issuance time that the selected user is still a service account, and refuses to issue against a human user. The issued application key is owned by, and scoped to, that service account. + +Revoking a service account application key requires the caller to supply that owning service account alongside the key itself. Until the requesting workflow threads it through, a revoke request that omits it fails rather than guessing which service account owns the key. *Schedules and application-key issuance are not enabled by default. Enable **Sync schedules** or **Sync secrets**, respectively, when configuring the connector. @@ -48,7 +50,7 @@ Configuring the connector requires you to pass in credentials generated in Datad A user with the **Connector Administrator** or **Super Administrator** role in C1 and the **Datadog Admin** or **Datadog standard** role in Datadog must perform this task. -If your user has a custom Datadog role, make sure it includes **User App Keys**, **User Access Invite**, and **User Access Manage** to create, update, enable, and disable users from C1. **User Access Manage** also governs issuing, syncing, and revoking service account application keys. If you enable **Sync secrets** to sync or revoke organization API keys, also add **API Keys Read**, **API Keys Write**, and **API Keys Delete**. +If your user has a custom Datadog role, make sure it includes **User App Keys**, **User Access Invite**, and **User Access Manage** to create, update, enable, and disable users from C1. **User Access Manage** also governs issuing, syncing, and revoking service account application keys. If you enable **Sync secrets** to sync or revoke organization API keys, also add **API Keys Read** and **API Keys Delete**. ### Locate your Datadog site diff --git a/pkg/client/client.go b/pkg/client/client.go index 156fd297..974f6a32 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -316,7 +316,12 @@ func (w *DatadogClient) FindServiceAccountApplicationKeyByName(ctx context.Conte ctx = w.withAuthContext(ctx) api := datadogV2.NewServiceAccountsApi(w.officialClient) const pageSize = int64(100) - for page := int64(0); ; page++ { + // maxPages bounds this loop so a provider that ignores page[number] and + // keeps returning full pages fails closed instead of spinning forever on + // the Issue hot path. 10_000 pages (1M keys) is far beyond any real + // service account's application-key count. + const maxPages = int64(10_000) + for page := int64(0); page < maxPages; page++ { params := *datadogV2.NewListServiceAccountApplicationKeysOptionalParameters().WithFilter(name).WithPageSize(pageSize).WithPageNumber(page) response, httpRes, err := api.ListServiceAccountApplicationKeys(ctx, serviceAccountID, params) if httpRes != nil { @@ -334,6 +339,7 @@ func (w *DatadogClient) FindServiceAccountApplicationKeyByName(ctx context.Conte return nil, nil } } + return nil, fmt.Errorf("find service account application key by name: exceeded %d pages without a short page", maxPages) } // ListServiceAccountApplicationKeys lists every application key owned by the diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 792d0260..d3072049 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "io" "net/http" "net/http/httptest" "strings" @@ -157,8 +158,7 @@ func TestServiceAccountApplicationKeyManagement(t *testing.T) { t.Run("create sends requested scopes", func(t *testing.T) { var body string server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - buf := make([]byte, r.ContentLength) - _, _ = r.Body.Read(buf) + buf, _ := io.ReadAll(r.Body) body = string(buf) w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"data":{"id":"appkey-id","type":"application_keys","attributes":{"key":"plaintext-app-key","name":"c1-request"}}}`)) diff --git a/pkg/connector/application_key.go b/pkg/connector/application_key.go index 915b261f..17a17421 100644 --- a/pkg/connector/application_key.go +++ b/pkg/connector/application_key.go @@ -125,7 +125,14 @@ func (o *applicationKeyBuilder) List( } serviceAccountResourceID := &v2.ResourceId{ResourceType: userResourceType.Id, Resource: serviceAccountID} - for appKeyPage := int64(0); ; appKeyPage++ { + // maxApplicationKeyPages bounds the inner drain so a provider that + // ignores page[number] and keeps returning full pages fails closed + // (an error, not an infinite request loop that never lets the SDK + // checkpoint). 10_000 pages (1M keys) is far beyond any real service + // account's application-key count. + const maxApplicationKeyPages = int64(10_000) + appKeyPage := int64(0) + for ; appKeyPage < maxApplicationKeyPages; appKeyPage++ { resp, err := o.wrapper.ListServiceAccountApplicationKeys(ctx, serviceAccountID, appKeyPage, defaultV2PageSize) if err != nil { return nil, nil, fmt.Errorf("baton-datadog: list application keys for service account %q: %w", serviceAccountID, err) @@ -145,6 +152,9 @@ func (o *applicationKeyBuilder) List( break } } + if appKeyPage >= maxApplicationKeyPages { + return nil, nil, fmt.Errorf("baton-datadog: exceeded %d application-key pages for service account %q without a short page", maxApplicationKeyPages, serviceAccountID) + } } nextPageToken := "" diff --git a/pkg/connector/credential_smoke_test.go b/pkg/connector/credential_smoke_test.go index 17ca5fe0..44253b94 100644 --- a/pkg/connector/credential_smoke_test.go +++ b/pkg/connector/credential_smoke_test.go @@ -7,6 +7,8 @@ import ( "testing" "time" + "github.com/DataDog/datadog-api-client-go/v2/api/datadog" + "github.com/conductorone/baton-datadog/pkg/client" cfg "github.com/conductorone/baton-datadog/pkg/config" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" @@ -15,11 +17,14 @@ import ( "google.golang.org/grpc/status" ) -// TestCredentialIssueLifecycle is an opt-in live-provider smoke test. It creates -// a real Datadog API key and always attempts to revoke it before returning. -// Run it only in a disposable Datadog organization: +// TestCredentialIssueLifecycle is an opt-in live-provider smoke test. It +// mints a real Datadog service-account application key and always attempts +// to revoke it before returning. Run it only in a disposable Datadog +// organization, against a service account that already exists there: // -// DATADOG_CREDENTIAL_SMOKE=1 DATADOG_SMOKE_SITE=datadoghq.com DATADOG_SMOKE_API_KEY=... DATADOG_SMOKE_APP_KEY=... \ +// DATADOG_CREDENTIAL_SMOKE=1 DATADOG_SMOKE_SITE=datadoghq.com \ +// DATADOG_SMOKE_API_KEY=... DATADOG_SMOKE_APP_KEY=... \ +// DATADOG_SMOKE_SERVICE_ACCOUNT_ID= \ // go test ./pkg/connector -run TestCredentialIssueLifecycle -count=1 func TestCredentialIssueLifecycle(t *testing.T) { if os.Getenv("DATADOG_CREDENTIAL_SMOKE") != "1" { @@ -29,9 +34,11 @@ func TestCredentialIssueLifecycle(t *testing.T) { site := os.Getenv("DATADOG_SMOKE_SITE") apiKey := os.Getenv("DATADOG_SMOKE_API_KEY") appKey := os.Getenv("DATADOG_SMOKE_APP_KEY") + serviceAccountID := os.Getenv("DATADOG_SMOKE_SERVICE_ACCOUNT_ID") require.NotEmpty(t, site, "DATADOG_SMOKE_SITE is required") require.NotEmpty(t, apiKey, "DATADOG_SMOKE_API_KEY is required") require.NotEmpty(t, appKey, "DATADOG_SMOKE_APP_KEY is required") + require.NotEmpty(t, serviceAccountID, "DATADOG_SMOKE_SERVICE_ACCOUNT_ID is required (issuance targets an existing Datadog service account)") ctx := context.Background() builder, _, err := New(ctx, &cfg.Datadog{ @@ -46,13 +53,14 @@ func TestCredentialIssueLifecycle(t *testing.T) { issuer := newCredentialUserBuilder(datadogConnector.wrapper) requestID := "smoke-" + time.Now().UTC().Format("20060102T150405") - t.Logf("issuing Datadog API key with request id %q", requestID) + t.Logf("issuing Datadog service account application key with request id %q", requestID) issued, err := issuer.Issue(ctx, &connectorbuilder.CredentialIssueInput{ IdentityID: &v2.ResourceId{ ResourceType: userResourceType.Id, - Resource: "credential-smoke-test", + Resource: serviceAccountID, }, - RequestID: requestID, + RequestID: requestID, + CredentialOptions: v2.CredentialIssueOptions_builder{ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build()}.Build(), }) require.NoError(t, err) revoked := false @@ -61,51 +69,81 @@ func TestCredentialIssueLifecycle(t *testing.T) { return } secretID := issued.Secret.GetId() - _, deleteErr := newApiTokenBuilder(datadogConnector.wrapper).Delete(ctx, secretID, nil) + parentID := issued.Secret.GetParentResourceId() + _, deleteErr := newApplicationKeyBuilder(datadogConnector.wrapper).Delete(ctx, secretID, parentID) if status.Code(deleteErr) != codes.NotFound { - require.NoError(t, deleteErr, "Datadog API key cleanup failed: %s", secretID.GetResource()) + require.NoError(t, deleteErr, "Datadog application key cleanup failed: %s", secretID.GetResource()) } }) require.NotNil(t, issued.Secret) require.NotEmpty(t, issued.Secret.GetId().GetResource()) + require.Equal(t, serviceAccountID, issued.Secret.GetParentResourceId().GetResource(), "issued secret must record the target service account as its parent resource") require.Equal(t, 1, len(issued.PlaintextData)) require.NotEmpty(t, issued.PlaintextData[0].GetBytes()) secretID := issued.Secret.GetId() - t.Logf("issued API key id=%s; plaintext material returned but not logged", maskedValue(secretID.GetResource())) - providerKey, err := datadogConnector.wrapper.GetAPIKey(ctx, secretID.GetResource()) - require.NoError(t, err, "read issued API key from Datadog") - providerKeyData := providerKey.GetData() - require.Equal(t, secretID.GetResource(), providerKeyData.GetId()) - t.Logf("confirmed API key id=%s exists in Datadog", maskedValue(secretID.GetResource())) - t.Logf("waiting for issued API key id=%s to propagate", maskedValue(secretID.GetResource())) + appKeyID := secretID.GetResource() + t.Logf("issued application key id=%s; plaintext material returned but not logged", maskedValue(appKeyID)) + + require.True(t, applicationKeyExists(t, ctx, datadogConnector.wrapper, serviceAccountID, appKeyID), + "issued application key id=%s not found via ListServiceAccountApplicationKeys", maskedValue(appKeyID)) + t.Logf("confirmed application key id=%s exists in Datadog", maskedValue(appKeyID)) + + t.Logf("waiting for issued application key id=%s to authenticate", maskedValue(appKeyID)) require.Eventually(t, func() bool { - issuedKeyValid, validateErr := datadogConnector.wrapper.ValidateAPIKey(ctx, string(issued.PlaintextData[0].GetBytes())) - return validateErr == nil && issuedKeyValid - }, 30*time.Second, time.Second, "issued API key did not become usable") - t.Logf("confirmed issued API key id=%s can authenticate with Datadog", maskedValue(secretID.GetResource())) + return canAuthenticate(ctx, site, apiKey, string(issued.PlaintextData[0].GetBytes())) + }, 30*time.Second, time.Second, "issued application key did not become usable") + t.Logf("confirmed issued application key id=%s can authenticate with Datadog", maskedValue(appKeyID)) - t.Logf("revoking API key id=%s", maskedValue(secretID.GetResource())) - _, err = newApiTokenBuilder(datadogConnector.wrapper).Delete(ctx, secretID, nil) - require.NoError(t, err, "revoke issued Datadog API key") - t.Logf("waiting for revoked API key id=%s to stop authenticating", maskedValue(secretID.GetResource())) + t.Logf("revoking application key id=%s", maskedValue(appKeyID)) + _, err = newApplicationKeyBuilder(datadogConnector.wrapper).Delete(ctx, secretID, issued.Secret.GetParentResourceId()) + require.NoError(t, err, "revoke issued Datadog application key") + t.Logf("waiting for revoked application key id=%s to stop authenticating", maskedValue(appKeyID)) require.Eventually(t, func() bool { - issuedKeyValid, validateErr := datadogConnector.wrapper.ValidateAPIKey(ctx, string(issued.PlaintextData[0].GetBytes())) - return !issuedKeyValid && (validateErr == nil || status.Code(validateErr) == codes.Unauthenticated || status.Code(validateErr) == codes.PermissionDenied) - }, 30*time.Second, time.Second, "revoked API key can still authenticate with Datadog") - t.Logf("confirmed revoked API key id=%s can no longer authenticate with Datadog", maskedValue(secretID.GetResource())) + return !canAuthenticate(ctx, site, apiKey, string(issued.PlaintextData[0].GetBytes())) + }, 30*time.Second, time.Second, "revoked application key can still authenticate with Datadog") + t.Logf("confirmed revoked application key id=%s can no longer authenticate with Datadog", maskedValue(appKeyID)) revoked = true - _, err = datadogConnector.wrapper.GetAPIKey(ctx, secretID.GetResource()) - if err == nil { - t.Logf("API key metadata id=%s remains retrievable after revocation; this does not imply the key can authenticate", maskedValue(secretID.GetResource())) + if applicationKeyExists(t, ctx, datadogConnector.wrapper, serviceAccountID, appKeyID) { + t.Logf("application key metadata id=%s remains listed after revocation; this does not imply the key can authenticate", maskedValue(appKeyID)) return } - if status.Code(err) == codes.NotFound { - t.Logf("confirmed API key id=%s is no longer retrievable from Datadog", maskedValue(secretID.GetResource())) - return + t.Logf("confirmed application key id=%s is no longer listed for its service account", maskedValue(appKeyID)) +} + +// applicationKeyExists checks the live provider for appKeyID among +// serviceAccountID's application keys, paging until found or exhausted. +func applicationKeyExists(t *testing.T, ctx context.Context, wrapper *client.DatadogClient, serviceAccountID, appKeyID string) bool { + t.Helper() + const maxPages = int64(10_000) + for page := int64(0); page < maxPages; page++ { + resp, err := wrapper.ListServiceAccountApplicationKeys(ctx, serviceAccountID, page, defaultV2PageSize) + require.NoError(t, err) + keys := resp.GetData() + for _, key := range keys { + if key.Id != nil && *key.Id == appKeyID { + return true + } + } + if int64(len(keys)) < defaultV2PageSize { + return false + } } - t.Logf("could not read API key metadata id=%s after revocation: %v", maskedValue(secretID.GetResource()), err) + t.Fatalf("exceeded %d pages listing application keys for service account %q without a short page", maxPages, serviceAccountID) + return false +} + +// canAuthenticate reports whether the given application key, paired with the +// smoke org's API key, can perform an authenticated read. Datadog has no +// application-key-only validation endpoint (unlike /api/v1/validate for API +// keys), so this performs a real authenticated request instead. +func canAuthenticate(ctx context.Context, site, apiKey, applicationKey string) bool { + cfg := datadog.NewConfiguration() + official := datadog.NewAPIClient(cfg) + probe := client.NewDatadogClient(nil, official, site, apiKey, applicationKey) + _, err := probe.ListTeams(ctx, nil) + return err == nil } func maskedValue(value string) string { diff --git a/pkg/connector/resource_types.go b/pkg/connector/resource_types.go index 68c18b2b..a8543ac3 100644 --- a/pkg/connector/resource_types.go +++ b/pkg/connector/resource_types.go @@ -49,7 +49,11 @@ var ( // them, but Issue no longer targets this type -- an org-scoped key // issued on behalf of a selected user is not an honest mapping of who // holds it. See serviceAccountApplicationKeyResourceType for the type - // Issue does target. + // Issue does target. No production path creates an org API key anymore, + // so this only declares api_keys_read/api_keys_delete, not + // api_keys_write -- CreateAPIKey/FindAPIKeyByName remain on + // DatadogClient for direct callers and tests, but this connector's own + // registered capabilities never exercise write. apiTokenResourceType = &v2.ResourceType{ Id: "api-key", DisplayName: "Organization API Key", @@ -57,7 +61,7 @@ var ( Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_SECRET}, Annotations: annotations.New( &v2.SkipEntitlementsAndGrants{}, - capabilityPermissions("api_keys_read", "api_keys_write", "api_keys_delete"), + capabilityPermissions("api_keys_read", "api_keys_delete"), ), } // serviceAccountApplicationKeyResourceType covers application keys owned diff --git a/pkg/connector/users.go b/pkg/connector/users.go index 8c3c7704..94100778 100644 --- a/pkg/connector/users.go +++ b/pkg/connector/users.go @@ -57,6 +57,9 @@ func (u *credentialUserBuilder) Issue(ctx context.Context, input *connectorbuild userResp, err := u.wrapper.GetUser(ctx, serviceAccountID) if err != nil { + if client.IsNotFound(err) { + return nil, status.Errorf(codes.NotFound, "baton-datadog: Datadog user %q not found", serviceAccountID) + } return nil, fmt.Errorf("baton-datadog: look up Datadog user %q: %w", serviceAccountID, err) } if !userResp.GetData().Attributes.GetServiceAccount() { From 5b951d5ac89c42225991db3e8c1d2a89790feb2d Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:47:15 +0000 Subject: [PATCH 22/49] fix: add api_keys_write to capabilities, bound FindAPIKeyByName, fix hasMoreAPIKeyPages, add tests Includes pre-existing fixes approved by owner: - Add api_keys_write to apiTokenResourceType (required by Datadog for DELETE) - Bound FindAPIKeyByName with 10k-page cap (mirrors other bounded helpers) - Fix hasMoreAPIKeyPages edge case when totalFilteredCount is zero - Fix gofmt issue at api_token.go:151 - Add CredentialIssuerV2 assertion on credentialUserBuilder - Add helpers_test.go table test for hasMoreAPIKeyPages Regenerated baton_capabilities.json with updated permissions. Co-authored-by: c1-squire-dev[bot] --- baton_capabilities.json | 6 ++++ pkg/client/client.go | 4 ++- pkg/connector/api_token.go | 4 +-- pkg/connector/helpers.go | 14 +++++--- pkg/connector/helpers_test.go | 60 +++++++++++++++++++++++++++++++++ pkg/connector/resource_types.go | 15 +++++---- pkg/connector/users.go | 1 + 7 files changed, 89 insertions(+), 15 deletions(-) create mode 100644 pkg/connector/helpers_test.go diff --git a/baton_capabilities.json b/baton_capabilities.json index a6351929..21a0877b 100644 --- a/baton_capabilities.json +++ b/baton_capabilities.json @@ -18,6 +18,9 @@ { "permission": "api_keys_read" }, + { + "permission": "api_keys_write" + }, { "permission": "api_keys_delete" } @@ -35,6 +38,9 @@ { "permission": "api_keys_read" }, + { + "permission": "api_keys_write" + }, { "permission": "api_keys_delete" } diff --git a/pkg/client/client.go b/pkg/client/client.go index 974f6a32..eddf56e3 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -202,7 +202,8 @@ func (w *DatadogClient) FindAPIKeyByName(ctx context.Context, name string) (*dat ctx = w.withAuthContext(ctx) api := datadogV2.NewKeyManagementApi(w.officialClient) const pageSize = int64(100) - for page := int64(0); ; page++ { + const maxPages = int64(10000) + for page := int64(0); page < maxPages; page++ { params := *datadogV2.NewListAPIKeysOptionalParameters().WithFilter(name).WithPageSize(pageSize).WithPageNumber(page) response, httpRes, err := api.ListAPIKeys(ctx, params) if httpRes != nil { @@ -220,6 +221,7 @@ func (w *DatadogClient) FindAPIKeyByName(ctx context.Context, name string) (*dat return nil, nil } } + return nil, fmt.Errorf("find API key by name: exceeded %d pages", maxPages) } func (w *DatadogClient) GetAPIKey(ctx context.Context, id string) (*datadogV2.APIKeyResponse, error) { diff --git a/pkg/connector/api_token.go b/pkg/connector/api_token.go index a38c3440..800fff31 100644 --- a/pkg/connector/api_token.go +++ b/pkg/connector/api_token.go @@ -147,8 +147,8 @@ func (o *apiTokenBuilder) List( if err != nil { return nil, nil, err } - ret = append(ret, rv) -} +ret = append(ret, rv) + } nextPageToken := "" if hasMoreAPIKeyPages(res, page, int64(len(apiTokens)), defaultV2PageSize) { diff --git a/pkg/connector/helpers.go b/pkg/connector/helpers.go index ba47dff7..a55125e5 100644 --- a/pkg/connector/helpers.go +++ b/pkg/connector/helpers.go @@ -120,9 +120,13 @@ func hasMoreAPIKeyPages(res *datadogV2.APIKeysResponse, page int64, count int64, if !ok || meta == nil { return count != 0 } - pageMeta := meta.GetPage() - total := pageMeta.GetTotalFilteredCount() - return page*pageSize+count < total + m := meta.GetPage() + if m.HasTotalFilteredCount() { + total := m.GetTotalFilteredCount() + if total == 0 { + return count != 0 + } + return page*pageSize+count < total + } + return count != 0 } - - diff --git a/pkg/connector/helpers_test.go b/pkg/connector/helpers_test.go new file mode 100644 index 00000000..1f5cbdcb --- /dev/null +++ b/pkg/connector/helpers_test.go @@ -0,0 +1,60 @@ +package connector + +import ( + "testing" + + "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" +) + +func TestHasMoreAPIKeyPages(t *testing.T) { + // Cases where meta is absent: fall back to count-based heuristic. + tests := []struct { + name string + res *datadogV2.APIKeysResponse + page int64 + count int64 + pageSize int64 + want bool + }{ + {name: "nil response with data", res: nil, page: 0, count: 10, pageSize: 100, want: true}, + {name: "nil response empty", res: nil, page: 0, count: 0, pageSize: 100, want: false}, + {name: "without meta full page", res: &datadogV2.APIKeysResponse{}, page: 0, count: 100, pageSize: 100, want: true}, + {name: "without meta partial page", res: &datadogV2.APIKeysResponse{}, page: 0, count: 50, pageSize: 100, want: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := hasMoreAPIKeyPages(tt.res, tt.page, tt.count, tt.pageSize) + if got != tt.want { + t.Errorf("hasMoreAPIKeyPages() = %v, want %v", got, tt.want) + } + }) + } + + // Authoritative total (HasTotalFilteredCount set to a real value). + metaWithTotal := func(total int64) *datadogV2.APIKeysResponse { + page := datadogV2.NewAPIKeysResponseMetaPage() + page.SetTotalFilteredCount(total) + return &datadogV2.APIKeysResponse{ + Meta: &datadogV2.APIKeysResponseMeta{Page: page}, + } + } + + if !hasMoreAPIKeyPages(metaWithTotal(250), 0, 100, 100) { + t.Error("page 0 of 250: expected more") + } + if !hasMoreAPIKeyPages(metaWithTotal(250), 1, 100, 100) { + t.Error("page 1 of 250: expected more") + } + if hasMoreAPIKeyPages(metaWithTotal(250), 2, 50, 100) { + t.Error("page 2 of 250 with 50: expected done") + } + + // Meta with nil page: count-based, same as "no meta". + nilPage := &datadogV2.APIKeysResponse{Meta: &datadogV2.APIKeysResponseMeta{}} + if !hasMoreAPIKeyPages(nilPage, 0, 100, 100) { + t.Error("nil page, 100 items: expected true (count-based)") + } + if hasMoreAPIKeyPages(nilPage, 0, 0, 100) { + t.Error("nil page, 0 items: expected false") + } +} diff --git a/pkg/connector/resource_types.go b/pkg/connector/resource_types.go index a8543ac3..46e8f225 100644 --- a/pkg/connector/resource_types.go +++ b/pkg/connector/resource_types.go @@ -49,11 +49,12 @@ var ( // them, but Issue no longer targets this type -- an org-scoped key // issued on behalf of a selected user is not an honest mapping of who // holds it. See serviceAccountApplicationKeyResourceType for the type - // Issue does target. No production path creates an org API key anymore, - // so this only declares api_keys_read/api_keys_delete, not - // api_keys_write -- CreateAPIKey/FindAPIKeyByName remain on - // DatadogClient for direct callers and tests, but this connector's own - // registered capabilities never exercise write. + // Issue does target. Sync/delete require api_keys_read/api_keys_delete; + // the DeleteAPIKey endpoint (DELETE /api/v2/api_keys/{api_key_id}) also + // requires api_keys_write per Datadog's documented permissions, so that + // is included here so C1 can gate the capability correctly at bind time. + // CreateAPIKey/FindAPIKeyByName remain on DatadogClient for direct + // callers and tests. apiTokenResourceType = &v2.ResourceType{ Id: "api-key", DisplayName: "Organization API Key", @@ -61,7 +62,7 @@ var ( Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_SECRET}, Annotations: annotations.New( &v2.SkipEntitlementsAndGrants{}, - capabilityPermissions("api_keys_read", "api_keys_delete"), + capabilityPermissions("api_keys_read", "api_keys_write", "api_keys_delete"), ), } // serviceAccountApplicationKeyResourceType covers application keys owned @@ -89,4 +90,4 @@ var ( capabilityPermissions("on_call_read"), ), } -) +) \ No newline at end of file diff --git a/pkg/connector/users.go b/pkg/connector/users.go index 94100778..0743cae9 100644 --- a/pkg/connector/users.go +++ b/pkg/connector/users.go @@ -111,6 +111,7 @@ var _ connectorbuilder.ResourceSyncerV2 = &userBuilder{} var _ connectorbuilder.AccountManagerV2 = &userBuilder{} var _ connectorbuilder.ResourceActionProvider = &userBuilder{} var _ connectorbuilder.CredentialIssuerLimited = &credentialUserBuilder{} +var _ connectorbuilder.CredentialIssuerV2 = &credentialUserBuilder{} var _ connectorbuilder.AccountManagerV2 = &credentialUserBuilder{} var _ connectorbuilder.ResourceActionProvider = &credentialUserBuilder{} From 9b9b6b21c0c85e71a813cc65ccbaedf504284d9d Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Sat, 22 Aug 2026 01:52:13 +0000 Subject: [PATCH 23/49] fix: run gofmt on connector files Fixes formatting on api_token.go (tab indent) and resource_types.go (trailing newline). Co-authored-by: c1-squire-dev[bot] --- pkg/connector/api_token.go | 2 +- pkg/connector/resource_types.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/connector/api_token.go b/pkg/connector/api_token.go index 800fff31..debbc50c 100644 --- a/pkg/connector/api_token.go +++ b/pkg/connector/api_token.go @@ -147,7 +147,7 @@ func (o *apiTokenBuilder) List( if err != nil { return nil, nil, err } -ret = append(ret, rv) + ret = append(ret, rv) } nextPageToken := "" diff --git a/pkg/connector/resource_types.go b/pkg/connector/resource_types.go index 46e8f225..60016872 100644 --- a/pkg/connector/resource_types.go +++ b/pkg/connector/resource_types.go @@ -90,4 +90,4 @@ var ( capabilityPermissions("on_call_read"), ), } -) \ No newline at end of file +) From 036b48968d832d5998a3499a2ede86bfd0bb2773 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Sat, 22 Aug 2026 02:15:13 +0000 Subject: [PATCH 24/49] fix: correct advertised Datadog permissions; page app keys one page per call The advertised Datadog permissions did not match what the endpoints this connector actually calls require, per the per-operation "x-permission" blocks in Datadog's own API spec and the published role-permission table: - DeleteAPIKey requires api_keys_delete. api_keys_write covers only CreateAPIKey/UpdateAPIKey ("Create and rename API Keys"), which no advertised capability on the api-key type calls, so it is dropped rather than making operators grant org-wide key-creation rights the connector never exercises. - Every service-account application-key endpoint the connector calls -- list (sync), create (issue) and delete (revoke) -- requires service_account_write. user_access_manage covers user disable, role management, SAML-to-role mappings and logs restriction queries, and grants none of the three, so sync/issue/revoke were advertised against a role Datadog answers with a 403. service_account_write is added to the service-account-application-key type and to the user type, where CAPABILITY_CREDENTIAL_ISSUE is registered. baton_capabilities.json is regenerated from the built binary rather than hand-edited, and the docs' custom-role guidance now names the same set. applicationKeyBuilder.List drained every application-key page for every service account inside a single call, buffering the whole org's keys before the SDK could checkpoint, respect rate limits or cancel. It now returns one provider page per call, keeping the users page and one child state per discovered service account in the pagination bag, and a 403 or 404 for a single service account is warned and skipped instead of failing the whole sync. Both behaviours are covered by new tests. Also drop GetAPIKey and ValidateAPIKey, which no production code and no test referenced after the smoke-test rework, and make the smoke test's canAuthenticate probe separate a credential rejection from a transient failure so the revocation assertion cannot pass on a 429 or a 5xx. Co-authored-by: c1-squire-dev[bot] --- baton_capabilities.json | 16 +- docs/connector.mdx | 6 +- pkg/client/client.go | 35 ---- pkg/connector/application_key.go | 163 ++++++++++++----- pkg/connector/credential_lifecycle_test.go | 202 +++++++++++++++++++++ pkg/connector/credential_smoke_test.go | 52 +++++- pkg/connector/resource_types.go | 45 ++++- 7 files changed, 417 insertions(+), 102 deletions(-) diff --git a/baton_capabilities.json b/baton_capabilities.json index 21a0877b..5fdaa44b 100644 --- a/baton_capabilities.json +++ b/baton_capabilities.json @@ -18,9 +18,6 @@ { "permission": "api_keys_read" }, - { - "permission": "api_keys_write" - }, { "permission": "api_keys_delete" } @@ -38,9 +35,6 @@ { "permission": "api_keys_read" }, - { - "permission": "api_keys_write" - }, { "permission": "api_keys_delete" } @@ -121,7 +115,7 @@ "@type": "type.googleapis.com/c1.connector.v2.CapabilityPermissions", "permissions": [ { - "permission": "user_access_manage" + "permission": "service_account_write" } ] } @@ -135,7 +129,7 @@ "permissions": { "permissions": [ { - "permission": "user_access_manage" + "permission": "service_account_write" } ] } @@ -189,6 +183,9 @@ }, { "permission": "user_access_manage" + }, + { + "permission": "service_account_write" } ] } @@ -206,6 +203,9 @@ }, { "permission": "user_access_manage" + }, + { + "permission": "service_account_write" } ] }, diff --git a/docs/connector.mdx b/docs/connector.mdx index 1018dc75..500f1393 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -19,7 +19,7 @@ sidebarTitle: "Datadog" | Teams | | | | | | Schedules | * | | | | | Secrets - Organization API keys | | | | | -| Secrets - Service account application keys | | | | | +| Secrets - Service account application keys | | | | † | [This connector can sync secrets](/product/admin/inventory) and display them on the **Inventory** page. Organization API keys and service account application keys are synced and shown as distinct secret kinds; only application keys owned by a Datadog service account can be issued. @@ -33,6 +33,8 @@ Revoking a service account application key requires the caller to supply that ow *Schedules and application-key issuance are not enabled by default. Enable **Sync schedules** or **Sync secrets**, respectively, when configuring the connector. +†Revoking a service account application key requires the request to name the owning service account as well as the key, because Datadog has no delete-by-key-id-alone form for these keys. A revoke request that omits it is refused rather than guessing. Until the requesting workflow supplies it, revocation is implemented and advertised but will not complete — see the note above. + ### Connector actions Connector actions are custom capabilities that extend C1 automations with app-specific operations. You can use connector actions in the [Perform connector action](/product/admin/automations-steps-reference#perform-connector-action) automation step. @@ -50,7 +52,7 @@ Configuring the connector requires you to pass in credentials generated in Datad A user with the **Connector Administrator** or **Super Administrator** role in C1 and the **Datadog Admin** or **Datadog standard** role in Datadog must perform this task. -If your user has a custom Datadog role, make sure it includes **User App Keys**, **User Access Invite**, and **User Access Manage** to create, update, enable, and disable users from C1. **User Access Manage** also governs issuing, syncing, and revoking service account application keys. If you enable **Sync secrets** to sync or revoke organization API keys, also add **API Keys Read** and **API Keys Delete**. +If your user has a custom Datadog role, make sure it includes **User App Keys**, **User Access Invite**, and **User Access Manage** to create, update, enable, and disable users from C1. If you enable **Sync secrets**, also add **Service Account Write**, which governs syncing, issuing, and revoking service account application keys, plus **API Keys Read** and **API Keys Delete** to sync and revoke organization API keys. ### Locate your Datadog site diff --git a/pkg/client/client.go b/pkg/client/client.go index eddf56e3..f2b5506c 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -224,41 +224,6 @@ func (w *DatadogClient) FindAPIKeyByName(ctx context.Context, name string) (*dat return nil, fmt.Errorf("find API key by name: exceeded %d pages", maxPages) } -func (w *DatadogClient) GetAPIKey(ctx context.Context, id string) (*datadogV2.APIKeyResponse, error) { - // GET /api/v2/api_keys/{api_key_id}. Requires the api_keys_read permission. - ctx = w.withAuthContext(ctx) - api := datadogV2.NewKeyManagementApi(w.officialClient) - response, httpRes, err := api.GetAPIKey(ctx, id) - if httpRes != nil { - defer httpRes.Body.Close() - } - if err != nil { - return nil, wrapOfficialClientError("get API key", httpRes, err) - } - return &response, nil -} - -func (w *DatadogClient) ValidateAPIKey(ctx context.Context, apiKey string) (bool, error) { - ctx = context.WithValue( - ctx, - datadog.ContextAPIKeys, - map[string]datadog.APIKey{ - "apiKeyAuth": {Key: apiKey}, - }, - ) - ctx = context.WithValue(ctx, datadog.ContextServerVariables, map[string]string{"site": w.site}) - // GET /api/v1/validate requires an API key and does not require an application key. - api := datadogV1.NewAuthenticationApi(w.officialClient) - response, httpRes, err := api.Validate(ctx) - if httpRes != nil { - defer httpRes.Body.Close() - } - if err != nil { - return false, wrapOfficialClientError("validate API key", httpRes, err) - } - return response.GetValid(), nil -} - func (w *DatadogClient) DeleteAPIKey(ctx context.Context, id string) error { ctx = w.withAuthContext(ctx) api := datadogV2.NewKeyManagementApi(w.officialClient) diff --git a/pkg/connector/application_key.go b/pkg/connector/application_key.go index 17a17421..8033a008 100644 --- a/pkg/connector/application_key.go +++ b/pkg/connector/application_key.go @@ -3,6 +3,7 @@ package connector import ( "context" "fmt" + "strconv" "time" "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" @@ -10,7 +11,10 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/conductorone/baton-sdk/pkg/pagination" "github.com/conductorone/baton-sdk/pkg/types/resource" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" ) @@ -92,13 +96,27 @@ func (o *applicationKeyBuilder) Delete(ctx context.Context, resourceID *v2.Resou return nil, nil } -// List syncs every application key owned by a Datadog service account. It -// pages through users (the same page cursor shape apiTokenBuilder/userBuilder -// use), and for each service account found in a page, fully drains that -// service account's own application-key pages via the dedicated +// maxApplicationKeyPages bounds one service account's application-key paging +// so a provider that ignores page[number] and keeps returning full pages +// fails closed instead of paging forever. 10_000 pages (1M keys) is far +// beyond any real service account's application-key count. +const maxApplicationKeyPages = int64(10_000) + +// List returns at most one provider page per call. The sync walks two levels: +// the users pages, to discover which users are service accounts, and then each +// service account's own application-key pages, read through the dedicated // service-account-scoped list API SPEC-07 requires -- not the org-wide -// application-key list, which would include human-owned keys this -// connector's issuance mapping deliberately never targets. +// application-key list, which would include human-owned keys this connector's +// issuance mapping deliberately never targets. +// +// Both levels live in the pagination bag: a users-level state at the bottom, +// and one child state per discovered service account pushed above it. Because +// the bag is a stack, a users page's service accounts are fully drained before +// the next users page is fetched, and every call issues exactly one provider +// list request. Draining every application-key page for every service account +// inside a single List call is the F2 pattern the connector criteria forbid -- +// it denies the SDK any chance to checkpoint, respect rate limits, or cancel, +// and buffers the whole org's keys in memory first. func (o *applicationKeyBuilder) List( ctx context.Context, _ *v2.ResourceId, @@ -109,13 +127,38 @@ func (o *applicationKeyBuilder) List( return nil, nil, err } + // A child state names the service account whose application keys it is + // paging; the users-level state carries no resource id. + if current := bag.Current(); current != nil && + current.ResourceTypeID == userResourceType.Id && current.ResourceID != "" { + return o.listApplicationKeyPage(ctx, bag, current.ResourceID, page) + } + return o.listServiceAccountsPage(ctx, bag, page) +} + +// listServiceAccountsPage consumes one users page and pushes a child state for +// every service account on it. It returns no resources of its own -- the +// application keys are produced by those child states on subsequent calls. +func (o *applicationKeyBuilder) listServiceAccountsPage( + ctx context.Context, + bag *pagination.Bag, + page int64, +) ([]*v2.Resource, *resource.SyncOpResults, error) { users, err := o.wrapper.ListUsers(ctx, datadogV2.NewListUsersOptionalParameters().WithPageNumber(page)) if err != nil { return nil, nil, fmt.Errorf("baton-datadog: list users while syncing service account application keys: %w", err) } - var ret []*v2.Resource - for _, user := range users.GetData() { + data := users.GetData() + if len(data) == 0 { + // Users are exhausted: drop the users-level state so the sync ends + // once the child states pushed by earlier pages are drained. + bag.Pop() + } else if err := bag.Next(strconv.FormatInt(page+1, 10)); err != nil { + return nil, nil, fmt.Errorf("baton-datadog: advance users page: %w", err) + } + + for _, user := range data { if user.Attributes == nil || !user.Attributes.GetServiceAccount() { continue } @@ -123,48 +166,84 @@ func (o *applicationKeyBuilder) List( if serviceAccountID == "" { continue } - serviceAccountResourceID := &v2.ResourceId{ResourceType: userResourceType.Id, Resource: serviceAccountID} - - // maxApplicationKeyPages bounds the inner drain so a provider that - // ignores page[number] and keeps returning full pages fails closed - // (an error, not an infinite request loop that never lets the SDK - // checkpoint). 10_000 pages (1M keys) is far beyond any real service - // account's application-key count. - const maxApplicationKeyPages = int64(10_000) - appKeyPage := int64(0) - for ; appKeyPage < maxApplicationKeyPages; appKeyPage++ { - resp, err := o.wrapper.ListServiceAccountApplicationKeys(ctx, serviceAccountID, appKeyPage, defaultV2PageSize) - if err != nil { - return nil, nil, fmt.Errorf("baton-datadog: list application keys for service account %q: %w", serviceAccountID, err) - } - keys := resp.GetData() - for _, key := range keys { - if key.Id == nil { - continue - } - rv, err := applicationKeyResource(*key.Id, serviceAccountResourceID, key.Attributes) - if err != nil { - return nil, nil, err - } - ret = append(ret, rv) - } - if int64(len(keys)) < defaultV2PageSize { - break + bag.Push(pagination.PageState{ + ResourceTypeID: userResourceType.Id, + ResourceID: serviceAccountID, + }) + } + + nextPageToken, err := bag.Marshal() + if err != nil { + return nil, nil, fmt.Errorf("baton-datadog: marshal pagination bag: %w", err) + } + return nil, &resource.SyncOpResults{NextPageToken: nextPageToken}, nil +} + +// listApplicationKeyPage returns one page of a single service account's +// application keys. +func (o *applicationKeyBuilder) listApplicationKeyPage( + ctx context.Context, + bag *pagination.Bag, + serviceAccountID string, + page int64, +) ([]*v2.Resource, *resource.SyncOpResults, error) { + if page >= maxApplicationKeyPages { + return nil, nil, fmt.Errorf( + "baton-datadog: exceeded %d application-key pages for service account %q without a short page", + maxApplicationKeyPages, serviceAccountID) + } + + resp, err := o.wrapper.ListServiceAccountApplicationKeys(ctx, serviceAccountID, page, defaultV2PageSize) + if err != nil { + // ListServiceAccountApplicationKeys requires Datadog's + // service_account_write permission, which this sync path is the + // first to need: an install that already had sync-secrets on for + // read-only key inventory may run a read-mostly custom role that + // lacks it. A service account can also be deleted mid-sync. Warn and + // skip that one service account rather than failing the whole sync + // (criteria R7); every other provider error still fails hard. + if code := status.Code(err); code == codes.PermissionDenied || code == codes.NotFound { + ctxzap.Extract(ctx).Warn( + "baton-datadog: skipping application keys for service account", + zap.String("service_account_id", serviceAccountID), + zap.String("code", code.String()), + zap.Error(err), + ) + bag.Pop() + nextPageToken, marshalErr := bag.Marshal() + if marshalErr != nil { + return nil, nil, fmt.Errorf("baton-datadog: marshal pagination bag: %w", marshalErr) } + return nil, &resource.SyncOpResults{NextPageToken: nextPageToken}, nil } - if appKeyPage >= maxApplicationKeyPages { - return nil, nil, fmt.Errorf("baton-datadog: exceeded %d application-key pages for service account %q without a short page", maxApplicationKeyPages, serviceAccountID) - } + return nil, nil, fmt.Errorf("baton-datadog: list application keys for service account %q: %w", serviceAccountID, err) } - nextPageToken := "" - if len(users.GetData()) != 0 { - nextPageToken, err = getPageTokenFromPage(bag, page+1) + serviceAccountResourceID := &v2.ResourceId{ResourceType: userResourceType.Id, Resource: serviceAccountID} + keys := resp.GetData() + ret := make([]*v2.Resource, 0, len(keys)) + for _, key := range keys { + if key.Id == nil { + continue + } + rv, err := applicationKeyResource(*key.Id, serviceAccountResourceID, key.Attributes) if err != nil { - return nil, nil, fmt.Errorf("baton-datadog: failed to get token from page: %w", err) + return nil, nil, err } + ret = append(ret, rv) } + if int64(len(keys)) < defaultV2PageSize { + // Short page: this service account is done. + bag.Pop() + } else if err := bag.Next(strconv.FormatInt(page+1, 10)); err != nil { + return nil, nil, fmt.Errorf("baton-datadog: advance application-key page: %w", err) + } + + nextPageToken, err := bag.Marshal() + if err != nil { + return nil, nil, fmt.Errorf("baton-datadog: marshal pagination bag: %w", err) + } return ret, &resource.SyncOpResults{NextPageToken: nextPageToken}, nil } diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index 73520ac3..cac6d1b2 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -3,6 +3,7 @@ package connector import ( "bytes" "context" + "fmt" "net/http" "net/http/httptest" "strings" @@ -12,6 +13,8 @@ import ( "github.com/conductorone/baton-datadog/pkg/client" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/conductorone/baton-sdk/pkg/pagination" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "github.com/stretchr/testify/require" "go.uber.org/zap" @@ -470,3 +473,202 @@ func TestApplicationKeyBuilderDeleteRejectsMalformedHandle(t *testing.T) { }) } } + +// --- applicationKeyBuilder.List paging ------------------------------------ + +// newAppKeyListServer fakes the two endpoints applicationKeyBuilder.List +// walks. usersPages[n] is the JSON "data" array for users page n, and +// appKeyPages[serviceAccountID][n] is that service account's application-key +// page n. A service account id present in forbidden gets a 403 instead, which +// is what a Datadog role without service_account_write returns. +func newAppKeyListServer( + t *testing.T, + usersPages []string, + appKeyPages map[string][]string, + forbidden map[string]bool, +) (*httptest.Server, *[]recordedRequest) { + t.Helper() + requests := &[]recordedRequest{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + recordRequest(t, requests, r) + w.Header().Set("Content-Type", "application/json") + page := 0 + if raw := r.URL.Query().Get("page[number]"); raw != "" { + if _, err := fmt.Sscanf(raw, "%d", &page); err != nil { + t.Errorf("unparsable page[number]=%q", raw) + } + } + + if r.URL.Path == "/api/v2/users" { + body := "[]" + if page < len(usersPages) { + body = usersPages[page] + } + _, _ = w.Write([]byte(`{"data":` + body + `}`)) + return + } + + const prefix = "/api/v2/service_accounts/" + const suffix = "/application_keys" + if strings.HasPrefix(r.URL.Path, prefix) && strings.HasSuffix(r.URL.Path, suffix) { + serviceAccountID := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, prefix), suffix) + if forbidden[serviceAccountID] { + w.WriteHeader(http.StatusForbidden) + _, _ = w.Write([]byte(`{"errors":["Forbidden"]}`)) + return + } + pages := appKeyPages[serviceAccountID] + body := "[]" + if page < len(pages) { + body = pages[page] + } + _, _ = w.Write([]byte(`{"data":` + body + `}`)) + return + } + + t.Errorf("unexpected provider request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + })) + return server, requests +} + +// appKeyPageJSON builds one application-key page of count keys, named by prefix. +func appKeyPageJSON(prefix string, count int) string { + keys := make([]string, 0, count) + for i := 0; i < count; i++ { + keys = append(keys, fmt.Sprintf(`{"id":"%s-%d","type":"application_keys","attributes":{"name":"%s-%d"}}`, prefix, i, prefix, i)) + } + return "[" + strings.Join(keys, ",") + "]" +} + +// drainAppKeyList runs List to exhaustion the way the SDK does -- feeding each +// call the previous call's NextPageToken -- and reports, per call, how many +// provider requests that single call issued. +func drainAppKeyList(t *testing.T, builder *applicationKeyBuilder, requests *[]recordedRequest) ([]*v2.Resource, []int) { + t.Helper() + ctx := context.Background() + var all []*v2.Resource + var requestsPerCall []int + token := "" + // The guard keeps a paging regression from hanging the suite. + const maxCalls = 100 + for call := 0; ; call++ { + require.Less(t, call, maxCalls, "List did not terminate") + before := len(*requests) + got, results, err := builder.List(ctx, nil, rs.SyncOpAttrs{PageToken: pagination.Token{Token: token}}) + require.NoError(t, err) + requestsPerCall = append(requestsPerCall, len(*requests)-before) + all = append(all, got...) + require.NotNil(t, results) + if results.NextPageToken == "" { + return all, requestsPerCall + } + token = results.NextPageToken + } +} + +// TestApplicationKeyBuilderListReturnsOnePagePerCall: List must issue at most +// one provider list request per call and must not drain a service account's +// application-key pages inside a single call (criteria F2), while still +// returning every key across the whole walk. +func TestApplicationKeyBuilderListReturnsOnePagePerCall(t *testing.T) { + usersPages := []string{ + `[{"id":"sa-1","type":"users","attributes":{"service_account":true}},` + + `{"id":"human-1","type":"users","attributes":{"service_account":false}},` + + `{"id":"sa-2","type":"users","attributes":{"service_account":true}}]`, + } + appKeyPages := map[string][]string{ + // sa-1 spans two pages: a full page forces a second request. + "sa-1": {appKeyPageJSON("sa1key", defaultV2PageSize), appKeyPageJSON("sa1key-p2", 1)}, + "sa-2": {appKeyPageJSON("sa2key", 2)}, + } + server, requests := newAppKeyListServer(t, usersPages, appKeyPages, nil) + defer server.Close() + + got, requestsPerCall := drainAppKeyList(t, newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)), requests) + + for call, n := range requestsPerCall { + require.LessOrEqualf(t, n, 1, "List call %d issued %d provider requests; at most one page per call is allowed", call, n) + } + // The walk needs a users page, three application-key pages (two for sa-1, + // one for sa-2) and a final empty users page. Spreading those over + // separate calls is the point: draining them inside one call is what F2 + // forbids, and would show up here as a single call issuing them all. + require.GreaterOrEqual(t, len(requestsPerCall), 5, "the walk must span multiple List calls, one provider page each") + require.Len(t, *requests, len(requestsPerCall), "one provider request per List call") + + require.Len(t, got, defaultV2PageSize+1+2, "every application key across both service accounts must be returned") + + ids := make(map[string]bool, len(got)) + for _, r := range got { + ids[r.GetId().GetResource()] = true + require.Equal(t, serviceAccountApplicationKeyResourceType.Id, r.GetId().GetResourceType()) + } + require.True(t, ids["sa1key-0"], "first page of sa-1 keys must be present") + require.True(t, ids["sa1key-p2-0"], "second page of sa-1 keys must be present") + require.True(t, ids["sa2key-0"], "sa-2 keys must be present") + + // A human user must never be queried for service-account application keys. + for _, req := range *requests { + require.NotContains(t, req.path, "human-1") + } +} + +// TestApplicationKeyBuilderListSkipsForbiddenServiceAccount: a 403 from +// ListServiceAccountApplicationKeys -- what an install whose Datadog role +// lacks service_account_write gets -- must skip that one service account and +// let the rest of the sync finish, not fail the whole sync (criteria R7). +func TestApplicationKeyBuilderListSkipsForbiddenServiceAccount(t *testing.T) { + usersPages := []string{ + `[{"id":"sa-forbidden","type":"users","attributes":{"service_account":true}},` + + `{"id":"sa-ok","type":"users","attributes":{"service_account":true}}]`, + } + appKeyPages := map[string][]string{"sa-ok": {appKeyPageJSON("okkey", 1)}} + server, requests := newAppKeyListServer(t, usersPages, appKeyPages, map[string]bool{"sa-forbidden": true}) + defer server.Close() + + got, _ := drainAppKeyList(t, newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)), requests) + + require.Len(t, got, 1, "the readable service account's keys must still sync") + require.Equal(t, "okkey-0", got[0].GetId().GetResource()) + + attempted := false + for _, req := range *requests { + if strings.Contains(req.path, "sa-forbidden") { + attempted = true + } + } + require.True(t, attempted, "the forbidden service account must actually have been attempted") +} + +// TestApplicationKeyBuilderListFailsHardOnUnexpectedError: only +// PermissionDenied/NotFound are skipped; any other provider error must still +// abort the sync rather than silently under-reporting keys. +func TestApplicationKeyBuilderListFailsHardOnUnexpectedError(t *testing.T) { + requests := &[]recordedRequest{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + recordRequest(t, requests, r) + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/api/v2/users" { + _, _ = w.Write([]byte(`{"data":[{"id":"sa-1","type":"users","attributes":{"service_account":true}}]}`)) + return + } + w.WriteHeader(http.StatusInternalServerError) + _, _ = w.Write([]byte(`{"errors":["boom"]}`)) + })) + defer server.Close() + + builder := newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)) + ctx := context.Background() + token := "" + for call := 0; call < 10; call++ { + _, results, err := builder.List(ctx, nil, rs.SyncOpAttrs{PageToken: pagination.Token{Token: token}}) + if err != nil { + return // expected: the 5xx aborted the sync + } + require.NotNil(t, results) + require.NotEmpty(t, results.NextPageToken, "sync ended without surfacing the provider 5xx") + token = results.NextPageToken + } + t.Fatal("List never surfaced the provider 5xx") +} diff --git a/pkg/connector/credential_smoke_test.go b/pkg/connector/credential_smoke_test.go index 44253b94..4aea63f8 100644 --- a/pkg/connector/credential_smoke_test.go +++ b/pkg/connector/credential_smoke_test.go @@ -91,7 +91,12 @@ func TestCredentialIssueLifecycle(t *testing.T) { t.Logf("waiting for issued application key id=%s to authenticate", maskedValue(appKeyID)) require.Eventually(t, func() bool { - return canAuthenticate(ctx, site, apiKey, string(issued.PlaintextData[0].GetBytes())) + ok, err := canAuthenticate(ctx, site, apiKey, string(issued.PlaintextData[0].GetBytes())) + if err != nil { + t.Logf("issued application key not usable yet: %v", err) + return false + } + return ok }, 30*time.Second, time.Second, "issued application key did not become usable") t.Logf("confirmed issued application key id=%s can authenticate with Datadog", maskedValue(appKeyID)) @@ -100,8 +105,20 @@ func TestCredentialIssueLifecycle(t *testing.T) { require.NoError(t, err, "revoke issued Datadog application key") t.Logf("waiting for revoked application key id=%s to stop authenticating", maskedValue(appKeyID)) require.Eventually(t, func() bool { - return !canAuthenticate(ctx, site, apiKey, string(issued.PlaintextData[0].GetBytes())) - }, 30*time.Second, time.Second, "revoked application key can still authenticate with Datadog") + ok, err := canAuthenticate(ctx, site, apiKey, string(issued.PlaintextData[0].GetBytes())) + if err != nil { + // Only Datadog refusing the credentials proves revocation. Any + // other error means the probe did not answer the question, so + // keep retrying instead of reading it as success. + if isCredentialRejection(err) { + return true + } + t.Logf("revocation probe failed without a credential rejection; retrying: %v", err) + return false + } + return !ok + }, 30*time.Second, time.Second, + "revoked application key still authenticates with Datadog, or the revocation probe never returned a credential rejection") t.Logf("confirmed revoked application key id=%s can no longer authenticate with Datadog", maskedValue(appKeyID)) revoked = true @@ -134,16 +151,37 @@ func applicationKeyExists(t *testing.T, ctx context.Context, wrapper *client.Dat return false } -// canAuthenticate reports whether the given application key, paired with the +// canAuthenticate probes whether the given application key, paired with the // smoke org's API key, can perform an authenticated read. Datadog has no // application-key-only validation endpoint (unlike /api/v1/validate for API // keys), so this performs a real authenticated request instead. -func canAuthenticate(ctx context.Context, site, apiKey, applicationKey string) bool { +// +// It reports three outcomes rather than two -- authenticated, or a specific +// failure -- so callers can tell the provider refusing the credentials apart +// from the probe simply not completing. Collapsing those (returning err == +// nil) would let the post-revoke assertion below, which is this PR's headline +// revocation evidence, pass on a transient network error, a 429 or a 5xx +// without the key ever having been revoked. +func canAuthenticate(ctx context.Context, site, apiKey, applicationKey string) (bool, error) { cfg := datadog.NewConfiguration() official := datadog.NewAPIClient(cfg) probe := client.NewDatadogClient(nil, official, site, apiKey, applicationKey) - _, err := probe.ListTeams(ctx, nil) - return err == nil + if _, err := probe.ListTeams(ctx, nil); err != nil { + return false, err + } + return true, nil +} + +// isCredentialRejection reports whether err is Datadog explicitly refusing the +// supplied credentials (401/403, mapped to Unauthenticated/PermissionDenied by +// wrapOfficialClientError) rather than any other failure. +func isCredentialRejection(err error) bool { + switch status.Code(err) { + case codes.Unauthenticated, codes.PermissionDenied: + return true + default: + return false + } } func maskedValue(value string) string { diff --git a/pkg/connector/resource_types.go b/pkg/connector/resource_types.go index 60016872..d2755ae9 100644 --- a/pkg/connector/resource_types.go +++ b/pkg/connector/resource_types.go @@ -15,6 +15,13 @@ func capabilityPermissions(perms ...string) *v2.CapabilityPermissions { } var ( + // userResourceType also carries CAPABILITY_CREDENTIAL_ISSUE: when + // sync-secrets is on, credentialUserBuilder is registered as the user + // syncer (see connector.go) and Issue mints a service-account + // application key. That path calls the service-account + // application-key endpoints, so service_account_write belongs here as + // well as on serviceAccountApplicationKeyResourceType -- without it C1 + // advertises issuance against a role that gets a 403 from Datadog. userResourceType = &v2.ResourceType{ Id: "user", DisplayName: "User", @@ -24,6 +31,7 @@ var ( capabilityPermissions( "user_access_invite", "user_access_manage", + "service_account_write", ), ), } @@ -49,12 +57,22 @@ var ( // them, but Issue no longer targets this type -- an org-scoped key // issued on behalf of a selected user is not an honest mapping of who // holds it. See serviceAccountApplicationKeyResourceType for the type - // Issue does target. Sync/delete require api_keys_read/api_keys_delete; - // the DeleteAPIKey endpoint (DELETE /api/v2/api_keys/{api_key_id}) also - // requires api_keys_write per Datadog's documented permissions, so that - // is included here so C1 can gate the capability correctly at bind time. - // CreateAPIKey/FindAPIKeyByName remain on DatadogClient for direct - // callers and tests. + // Issue does target. + // + // The advertised permissions are the ones Datadog's own API spec marks + // required (the per-operation "x-permission" block) for the only two + // endpoints the advertised capabilities call: ListAPIKeys, backing + // CAPABILITY_SYNC, requires api_keys_read, and DeleteAPIKey + // (DELETE /api/v2/api_keys/{api_key_id}), backing + // CAPABILITY_RESOURCE_DELETE, requires api_keys_delete. api_keys_delete + // is a real Datadog permission ("API Keys Delete -- Delete API Keys for + // your organization", Datadog Admin Role) and is the one that governs + // delete; api_keys_write is scoped to CreateAPIKey/UpdateAPIKey ("Create + // and rename API Keys") and is deliberately NOT advertised here, because + // no advertised capability on this type creates or renames a key. + // Advertising it would make C1 demand org-wide key-creation rights the + // connector never exercises. CreateAPIKey/FindAPIKeyByName remain on + // DatadogClient for tests. apiTokenResourceType = &v2.ResourceType{ Id: "api-key", DisplayName: "Organization API Key", @@ -62,7 +80,7 @@ var ( Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_SECRET}, Annotations: annotations.New( &v2.SkipEntitlementsAndGrants{}, - capabilityPermissions("api_keys_read", "api_keys_write", "api_keys_delete"), + capabilityPermissions("api_keys_read", "api_keys_delete"), ), } // serviceAccountApplicationKeyResourceType covers application keys owned @@ -72,6 +90,17 @@ var ( // resource of this type is distinguishable from an apiTokenResourceType // (organization API key) by resource type id, display name, and the // underlying SecretTrait's credential_detail (see application_key.go). + // + // Every service-account application-key endpoint this connector calls is + // marked service_account_write in Datadog's API spec: the + // ListServiceAccountApplicationKeys sync path, the + // CreateServiceAccountApplicationKey issue path, and the + // DeleteServiceAccountApplicationKey revoke path. Datadog describes that + // permission as "Create, disable, and use Service Accounts in your + // organization" (Datadog Admin Role). user_access_manage covers user + // disable, role management, SAML-to-role mappings and logs restriction + // queries -- it grants none of the three, so advertising it here would + // let C1 offer sync/issue/revoke to a role Datadog answers with a 403. serviceAccountApplicationKeyResourceType = &v2.ResourceType{ Id: "service-account-application-key", DisplayName: "Service Account Application Key", @@ -79,7 +108,7 @@ var ( Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_SECRET}, Annotations: annotations.New( &v2.SkipEntitlementsAndGrants{}, - capabilityPermissions("user_access_manage"), + capabilityPermissions("service_account_write"), ), } scheduleResourceType = &v2.ResourceType{ From 0228d6eb04b12f4ccb5e62a24d3c075cbb052c81 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Sat, 22 Aug 2026 03:52:39 +0000 Subject: [PATCH 25/49] fix: scope service_account_write to the secrets types; sample the skip warning Three follow-ups on the permission metadata, all in the surfaces the previous commit touched. Drop service_account_write from userResourceType. That type is registered on every install, but issuance only exists when sync-secrets is on -- the same flag that swaps credentialUserBuilder in for userBuilder and registers applicationKeyBuilder at all -- so a sync-secrets-off install was being told to grant a Datadog Admin permission no code path in that configuration can reach. baton_capabilities.json cannot express the condition directly: it is one static document generated from a connector built with SyncSecrets and SyncSchedules forced true (cmd/baton-datadog/main.go), and CapabilityPermissions has no conditional form. Scoping the permission to serviceAccountApplicationKeyResourceType, which is only registered under the flag, is how the conditionality is carried; the user type's credential_issue block already points at that type via secretResourceTypeId, so the requirement stays discoverable. This also makes the metadata agree with what docs/connector.mdx already said. Confirm service_account_write as the permission for the service-account application-key endpoints, and do not also advertise user_access_manage. Datadog's published OpenAPI spec gives each operation an "x-permission" OR list, and for ListServiceAccountApplicationKeys, CreateServiceAccountApplicationKey and DeleteServiceAccountApplicationKey that list has exactly one entry. The same spec does name several accepted permissions where several are accepted -- DisableUser and UpdateUser accept user_access_manage OR service_account_write -- so the single-entry list is meaningful rather than an omission. user_access_manage is an AuthZ oauth2 scope, a different axis from RBAC that does not apply to a connector authenticating with apiKeyAuth and appKeyAuth. Recorded in the type's doc comment so the next reader does not have to re-derive it. Sample the per-service-account skip warning. The case it exists for is a role missing service_account_write org-wide, which made it fire once per service account on every sync. It now logs the 1st, 10th and 100th occurrence and every 1000th after that, with a total_occurrences field, per criteria L7. Neither this repo nor the vendored baton-sdk ships a sampling helper, so shouldLogSampled is the smallest thing that satisfies the criteria: a pure function of the count, with the counter owned by the builder. Co-authored-by: c1-squire-dev[bot] --- baton_capabilities.json | 6 -- pkg/connector/application_key.go | 27 +++++-- pkg/connector/credential_lifecycle_test.go | 82 ++++++++++++++++++++++ pkg/connector/helpers.go | 21 ++++++ pkg/connector/resource_types.go | 67 +++++++++++++----- 5 files changed, 173 insertions(+), 30 deletions(-) diff --git a/baton_capabilities.json b/baton_capabilities.json index 5fdaa44b..32249a6e 100644 --- a/baton_capabilities.json +++ b/baton_capabilities.json @@ -183,9 +183,6 @@ }, { "permission": "user_access_manage" - }, - { - "permission": "service_account_write" } ] } @@ -203,9 +200,6 @@ }, { "permission": "user_access_manage" - }, - { - "permission": "service_account_write" } ] }, diff --git a/pkg/connector/application_key.go b/pkg/connector/application_key.go index 8033a008..5d42ace2 100644 --- a/pkg/connector/application_key.go +++ b/pkg/connector/application_key.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strconv" + "sync/atomic" "time" "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" @@ -22,6 +23,13 @@ import ( type applicationKeyBuilder struct { resourceType *v2.ResourceType wrapper *client.DatadogClient + + // skippedServiceAccounts counts how many service accounts this syncer has + // skipped because their application keys could not be read. The warning + // that reports a skip is sampled (L7): the case it exists for is a role + // missing service_account_write org-wide, which makes it fire for every + // service account on every sync. + skippedServiceAccounts atomic.Int64 } var _ connectorbuilder.ResourceSyncerV2 = &applicationKeyBuilder{} @@ -203,12 +211,19 @@ func (o *applicationKeyBuilder) listApplicationKeyPage( // skip that one service account rather than failing the whole sync // (criteria R7); every other provider error still fails hard. if code := status.Code(err); code == codes.PermissionDenied || code == codes.NotFound { - ctxzap.Extract(ctx).Warn( - "baton-datadog: skipping application keys for service account", - zap.String("service_account_id", serviceAccountID), - zap.String("code", code.String()), - zap.Error(err), - ) + // Sampled, not per-service-account: an org-wide missing + // service_account_write would otherwise emit one line per service + // account on every sync. total_occurrences keeps the real count + // visible on the lines that do get through. + if total := o.skippedServiceAccounts.Add(1); shouldLogSampled(total) { + ctxzap.Extract(ctx).Warn( + "baton-datadog: skipping application keys for service account", + zap.String("service_account_id", serviceAccountID), + zap.String("code", code.String()), + zap.Int64("total_occurrences", total), + zap.Error(err), + ) + } bag.Pop() nextPageToken, marshalErr := bag.Marshal() if marshalErr != nil { diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index cac6d1b2..f93603fc 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -672,3 +672,85 @@ func TestApplicationKeyBuilderListFailsHardOnUnexpectedError(t *testing.T) { } t.Fatal("List never surfaced the provider 5xx") } + +// TestShouldLogSampled: the L7 schedule is the 1st, 10th and 100th occurrence, +// then every 1000th, and nothing else. +func TestShouldLogSampled(t *testing.T) { + logged := map[int64]bool{} + for n := int64(1); n <= 3000; n++ { + if shouldLogSampled(n) { + logged[n] = true + } + } + for _, want := range []int64{1, 10, 100, 1000, 2000, 3000} { + require.Truef(t, logged[want], "occurrence %d should be logged", want) + } + for _, notWant := range []int64{2, 9, 11, 99, 101, 999, 1001, 1999} { + require.Falsef(t, logged[notWant], "occurrence %d should not be logged", notWant) + } + require.Len(t, logged, 6, "exactly 1, 10, 100, 1000, 2000, 3000 in the first 3000") + require.False(t, shouldLogSampled(0), "a zero count is not an occurrence") + require.False(t, shouldLogSampled(-1), "a negative count is not an occurrence") +} + +// TestApplicationKeyBuilderListSamplesSkipWarning: when the configured Datadog +// role cannot read application keys org-wide, the skip warning must not fire +// once per service account (criteria L7). With 12 forbidden service accounts +// only the 1st and 10th are logged, and each surviving line carries +// total_occurrences so the real count is still visible. +func TestApplicationKeyBuilderListSamplesSkipWarning(t *testing.T) { + const serviceAccounts = 12 + entries := make([]string, 0, serviceAccounts) + forbidden := map[string]bool{} + for i := 0; i < serviceAccounts; i++ { + id := fmt.Sprintf("sa-%02d", i) + entries = append(entries, fmt.Sprintf(`{"id":"%s","type":"users","attributes":{"service_account":true}}`, id)) + forbidden[id] = true + } + usersPages := []string{"[" + strings.Join(entries, ",") + "]"} + + server, requests := newAppKeyListServer(t, usersPages, nil, forbidden) + defer server.Close() + + var logBuf bytes.Buffer + core := zapcore.NewCore( + zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(&logBuf), + zapcore.DebugLevel, + ) + logger := zap.New(core) + ctx := ctxzap.ToContext(context.Background(), logger) + + builder := newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)) + token := "" + for call := 0; call < 100; call++ { + _, results, err := builder.List(ctx, nil, rs.SyncOpAttrs{PageToken: pagination.Token{Token: token}}) + require.NoError(t, err, "a 403 must never fail the sync") + require.NotNil(t, results) + if results.NextPageToken == "" { + break + } + token = results.NextPageToken + } + require.NoError(t, logger.Sync()) + + // Every service account was still attempted and skipped. + attempted := 0 + for _, req := range *requests { + if strings.Contains(req.path, "/application_keys") { + attempted++ + } + } + require.Equal(t, serviceAccounts, attempted, "every service account must still be attempted") + + skipLines := 0 + for _, line := range strings.Split(strings.TrimSpace(logBuf.String()), "\n") { + if line == "" || !strings.Contains(line, "skipping application keys for service account") { + continue + } + skipLines++ + require.Contains(t, line, "total_occurrences", "a sampled warning must report the real count") + } + require.Equal(t, 2, skipLines, "12 skips must log only the 1st and 10th, not one line each") + require.Contains(t, logBuf.String(), `"total_occurrences":10`, "the 10th occurrence must be the second logged line") +} diff --git a/pkg/connector/helpers.go b/pkg/connector/helpers.go index a55125e5..3762df2a 100644 --- a/pkg/connector/helpers.go +++ b/pkg/connector/helpers.go @@ -130,3 +130,24 @@ func hasMoreAPIKeyPages(res *datadogV2.APIKeysResponse, page int64, count int64, } return count != 0 } + +// shouldLogSampled reports whether the nth occurrence (1-based) of a repeating +// event should be logged, on the logarithmic schedule the repo's review +// criteria require for warnings that can fire once per resource (L7): the 1st, +// 10th and 100th occurrence, then every 1000th. Callers pass the running total +// and put it on the record as total_occurrences, so a sampled line still says +// how many times the event really happened. +// +// Neither this repo nor the vendored baton-sdk ships a sampling helper, so this +// is the smallest thing that satisfies the criteria; it is deliberately a pure +// function of the count, with the counter owned by the caller. +func shouldLogSampled(n int64) bool { + switch { + case n <= 0: + return false + case n == 1, n == 10, n == 100: + return true + default: + return n%1000 == 0 + } +} diff --git a/pkg/connector/resource_types.go b/pkg/connector/resource_types.go index d2755ae9..94f5fa64 100644 --- a/pkg/connector/resource_types.go +++ b/pkg/connector/resource_types.go @@ -15,13 +15,29 @@ func capabilityPermissions(perms ...string) *v2.CapabilityPermissions { } var ( - // userResourceType also carries CAPABILITY_CREDENTIAL_ISSUE: when - // sync-secrets is on, credentialUserBuilder is registered as the user - // syncer (see connector.go) and Issue mints a service-account - // application key. That path calls the service-account - // application-key endpoints, so service_account_write belongs here as - // well as on serviceAccountApplicationKeyResourceType -- without it C1 - // advertises issuance against a role that gets a 403 from Datadog. + // userResourceType is registered on every install (connector.go), so its + // advertised permissions must be the ones the always-on user sync and + // account provisioning actually call: user_access_invite for CreateUser, + // user_access_manage for UpdateUser/DisableUser. + // + // service_account_write is deliberately NOT listed here even though + // userResourceType carries CAPABILITY_CREDENTIAL_ISSUE. Issuance only + // exists when sync-secrets is on -- that is the flag that swaps + // credentialUserBuilder in for userBuilder and registers + // applicationKeyBuilder at all -- so listing it here would tell every + // sync-secrets-off install to grant a Datadog Admin permission that no + // code path in that configuration can reach. + // + // baton_capabilities.json cannot express "only when sync-secrets is on": + // it is one static document, generated by `./connector capabilities` from + // a connector built with SyncSecrets and SyncSchedules forced true (see + // cmd/baton-datadog/main.go), and CapabilityPermissions has no + // conditional form. Scoping the permission to the resource type that only + // exists under the flag is therefore how the conditionality is carried: + // service_account_write lives on serviceAccountApplicationKeyResourceType, + // which is only registered when sync-secrets is on, and the user type's + // credential_issue block points at that type via secretResourceTypeId, so + // the requirement is still discoverable from the metadata. userResourceType = &v2.ResourceType{ Id: "user", DisplayName: "User", @@ -31,7 +47,6 @@ var ( capabilityPermissions( "user_access_invite", "user_access_manage", - "service_account_write", ), ), } @@ -91,16 +106,32 @@ var ( // (organization API key) by resource type id, display name, and the // underlying SecretTrait's credential_detail (see application_key.go). // - // Every service-account application-key endpoint this connector calls is - // marked service_account_write in Datadog's API spec: the - // ListServiceAccountApplicationKeys sync path, the - // CreateServiceAccountApplicationKey issue path, and the - // DeleteServiceAccountApplicationKey revoke path. Datadog describes that - // permission as "Create, disable, and use Service Accounts in your - // organization" (Datadog Admin Role). user_access_manage covers user - // disable, role management, SAML-to-role mappings and logs restriction - // queries -- it grants none of the three, so advertising it here would - // let C1 offer sync/issue/revoke to a role Datadog answers with a 403. + // Every service-account application-key endpoint this connector calls + // requires exactly one Datadog RBAC permission, service_account_write: + // ListServiceAccountApplicationKeys (sync), + // CreateServiceAccountApplicationKey (issue) and + // DeleteServiceAccountApplicationKey (revoke). This is the "x-permission" + // block Datadog publishes for each operation in its own OpenAPI spec + // (docs.datadoghq.com/resources/json/full_spec_v2.json, the spec that + // renders the public API reference); Datadog's role-permission page + // describes the permission as "Create, disable, and use Service Accounts + // in your organization" (Datadog Admin Role). + // + // Only service_account_write is advertised, not user_access_manage as + // well, and that is deliberate. The spec's x-permission lists are OR + // lists that do name several accepted permissions when several are + // accepted -- DisableUser and UpdateUser, for instance, accept + // user_access_manage OR service_account_write. For the application-key + // operations the list has exactly one entry, so user_access_manage is not + // an accepted alternative and advertising it would send operators after a + // permission the API will not honour for these calls. + // + // user_access_manage does appear in Datadog's AuthZ oauth2 scope list, + // which is a different axis from RBAC and does not apply here: this + // connector authenticates with apiKeyAuth + appKeyAuth, and every value + // in this file is an RBAC permission id. service_account_write is not an + // OAuth scope at all, so an OAuth-scope reading of these endpoints will + // disagree with this annotation by design. serviceAccountApplicationKeyResourceType = &v2.ResourceType{ Id: "service-account-application-key", DisplayName: "Service Account Application Key", From 87c20bb192d6d9f8abef61c71e570b42fc4ef112 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Sat, 22 Aug 2026 04:02:18 +0000 Subject: [PATCH 26/49] fix: reset the skip-warning sample counter at the start of each sync applicationKeyBuilder is constructed once per connector process -- Datadog.ResourceSyncers runs inside connectorbuilder.NewConnector, which the SDK calls once at startup and whose registered syncers are reused for every sync -- so skippedServiceAccounts never reset. The first sync consumed the 1st, 10th and 100th log slots, and a second sync against the same org-wide missing service_account_write incremented the total while emitting nothing at all until it reached 1000. That is worse than the per-resource noise the sampling was added to prevent. List now zeroes the counter when the page token is empty, giving each walk its own schedule. An empty token is a safe first-walk signal here: List only returns an empty NextPageToken from bag.Marshal() with an empty bag, and the bag can only be emptied by popping the users-level state, which happens solely on an empty users page -- the end of the walk -- so no mid-walk call can carry one. A retried first page resets again, which is what a restarted walk wants. TestApplicationKeyBuilderListSamplesSkipWarning now drains twice against the same builder and asserts each walk logs its own occurrence 1 and 10. Without the reset the second walk logs nothing, so the assertion discriminates. The drain helper also asserts no mid-walk token is empty, pinning the invariant the reset relies on. Co-authored-by: c1-squire-dev[bot] --- pkg/connector/application_key.go | 28 ++++++- pkg/connector/credential_lifecycle_test.go | 90 ++++++++++++++-------- 2 files changed, 82 insertions(+), 36 deletions(-) diff --git a/pkg/connector/application_key.go b/pkg/connector/application_key.go index 5d42ace2..c2fa07c6 100644 --- a/pkg/connector/application_key.go +++ b/pkg/connector/application_key.go @@ -24,11 +24,12 @@ type applicationKeyBuilder struct { resourceType *v2.ResourceType wrapper *client.DatadogClient - // skippedServiceAccounts counts how many service accounts this syncer has - // skipped because their application keys could not be read. The warning + // skippedServiceAccounts counts how many service accounts the current walk + // has skipped because their application keys could not be read. The warning // that reports a skip is sampled (L7): the case it exists for is a role // missing service_account_write org-wide, which makes it fire for every - // service account on every sync. + // service account. It is reset at the start of each walk (see List) because + // this builder outlives any single sync. skippedServiceAccounts atomic.Int64 } @@ -130,6 +131,27 @@ func (o *applicationKeyBuilder) List( _ *v2.ResourceId, opts resource.SyncOpAttrs, ) ([]*v2.Resource, *resource.SyncOpResults, error) { + // An empty page token is the first call of a walk, so the skip-warning + // sampling counter restarts here and each sync gets its own 1/10/100 + // schedule. The builder is constructed once per connector process -- + // Datadog.ResourceSyncers runs inside connectorbuilder.NewConnector, whose + // result is reused for every sync -- so without this reset the first sync + // consumes the early log slots and a later sync against the same org-wide + // missing permission would emit nothing until the running total reached + // 1000. + // + // An empty token is a safe first-walk signal for this builder: List only + // ever returns an empty NextPageToken from bag.Marshal() with an empty + // bag, and the bag can only be emptied by popping the users-level state, + // which happens solely on an empty users page -- the end of the walk. So + // no mid-walk call can carry one. (parsePageToken also accepts a "page:N" + // seed form, which nothing produces for this resource type; if one ever + // did, the counter would merely carry over rather than misbehave.) A + // retried first page resets again, which is what a restarted walk wants. + if opts.PageToken.Token == "" { + o.skippedServiceAccounts.Store(0) + } + bag, page, err := parsePageToken(opts.PageToken.Token, &v2.ResourceId{ResourceType: o.resourceType.Id}) if err != nil { return nil, nil, err diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index f93603fc..d18956f8 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -693,11 +693,56 @@ func TestShouldLogSampled(t *testing.T) { require.False(t, shouldLogSampled(-1), "a negative count is not an occurrence") } +// drainForSkipWarnings runs one full List walk with its own log sink and +// returns the skip-warning lines that walk emitted. +func drainForSkipWarnings(t *testing.T, builder *applicationKeyBuilder) []string { + t.Helper() + var logBuf bytes.Buffer + core := zapcore.NewCore( + zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(&logBuf), + zapcore.DebugLevel, + ) + logger := zap.New(core) + ctx := ctxzap.ToContext(context.Background(), logger) + + token := "" + for call := 0; call < 200; call++ { + require.Less(t, call, 199, "List did not terminate") + _, results, err := builder.List(ctx, nil, rs.SyncOpAttrs{PageToken: pagination.Token{Token: token}}) + require.NoError(t, err, "a 403 must never fail the sync") + require.NotNil(t, results) + if results.NextPageToken == "" { + break + } + // Only the first call of a walk may carry an empty token; the reset in + // List depends on that, so assert it rather than assuming it. + require.NotEmpty(t, results.NextPageToken, "mid-walk token must never be empty") + token = results.NextPageToken + } + require.NoError(t, logger.Sync()) + + var skips []string + for _, line := range strings.Split(strings.TrimSpace(logBuf.String()), "\n") { + if line != "" && strings.Contains(line, "skipping application keys for service account") { + skips = append(skips, line) + } + } + return skips +} + // TestApplicationKeyBuilderListSamplesSkipWarning: when the configured Datadog // role cannot read application keys org-wide, the skip warning must not fire // once per service account (criteria L7). With 12 forbidden service accounts -// only the 1st and 10th are logged, and each surviving line carries -// total_occurrences so the real count is still visible. +// only the 1st and 10th are logged, each carrying total_occurrences. +// +// The counter also has to restart per walk. applicationKeyBuilder is built once +// per connector process (Datadog.ResourceSyncers runs inside +// connectorbuilder.NewConnector), so a counter that only ever climbed would let +// the first sync consume the 1/10/100 slots and leave every later sync silent +// until the running total reached 1000 -- which is worse than the noise the +// sampling exists to prevent. Draining twice proves the second walk gets its +// own schedule. func TestApplicationKeyBuilderListSamplesSkipWarning(t *testing.T) { const serviceAccounts = 12 entries := make([]string, 0, serviceAccounts) @@ -712,45 +757,24 @@ func TestApplicationKeyBuilderListSamplesSkipWarning(t *testing.T) { server, requests := newAppKeyListServer(t, usersPages, nil, forbidden) defer server.Close() - var logBuf bytes.Buffer - core := zapcore.NewCore( - zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), - zapcore.AddSync(&logBuf), - zapcore.DebugLevel, - ) - logger := zap.New(core) - ctx := ctxzap.ToContext(context.Background(), logger) - builder := newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)) - token := "" - for call := 0; call < 100; call++ { - _, results, err := builder.List(ctx, nil, rs.SyncOpAttrs{PageToken: pagination.Token{Token: token}}) - require.NoError(t, err, "a 403 must never fail the sync") - require.NotNil(t, results) - if results.NextPageToken == "" { - break + + for walk := 1; walk <= 2; walk++ { + skips := drainForSkipWarnings(t, builder) + require.Lenf(t, skips, 2, "walk %d: 12 skips must log only the 1st and 10th, not one line each", walk) + for _, line := range skips { + require.Containsf(t, line, "total_occurrences", "walk %d: a sampled warning must report the real count", walk) } - token = results.NextPageToken + require.Containsf(t, skips[0], `"total_occurrences":1`, "walk %d: first logged line is occurrence 1", walk) + require.Containsf(t, skips[1], `"total_occurrences":10`, "walk %d: second logged line is occurrence 10", walk) } - require.NoError(t, logger.Sync()) - // Every service account was still attempted and skipped. + // Both walks attempted every service account. attempted := 0 for _, req := range *requests { if strings.Contains(req.path, "/application_keys") { attempted++ } } - require.Equal(t, serviceAccounts, attempted, "every service account must still be attempted") - - skipLines := 0 - for _, line := range strings.Split(strings.TrimSpace(logBuf.String()), "\n") { - if line == "" || !strings.Contains(line, "skipping application keys for service account") { - continue - } - skipLines++ - require.Contains(t, line, "total_occurrences", "a sampled warning must report the real count") - } - require.Equal(t, 2, skipLines, "12 skips must log only the 1st and 10th, not one line each") - require.Contains(t, logBuf.String(), `"total_occurrences":10`, "the 10th occurrence must be the second logged line") + require.Equal(t, serviceAccounts*2, attempted, "every service account must be attempted on every walk") } From 819144ef6dc7709f33ff9a476408e13fadf762a9 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Sat, 22 Aug 2026 04:13:25 +0000 Subject: [PATCH 27/49] test: make the skip-warning assertions capable of failing Two assertions added with the sampling tests could not fail, which is worse than no assertion because it reads as coverage. require.NotEmpty on results.NextPageToken sat three lines below an `if results.NextPageToken == "" { break }`, so the only input that could have failed it had already left the loop. The invariant it claimed to pin -- that an empty token means first-call-of-this-walk, which is what List's counter reset depends on -- was asserted nowhere. The check now lives in the break branch and asserts the walk did not terminate on call zero, which is the property that makes an empty token unambiguous between start and end. Early termination partway through a walk is caught separately by the attempted-request count. The occurrence assertion compared the log line against the substring `"total_occurrences":1`, which is a prefix of :10, :100 and :1000, so a walk whose first sampled line was a later occurrence would still have passed -- and that is precisely the property the per-walk reset exists to prove. Skip warnings are now decoded from the JSON records and total_occurrences is compared numerically, so 1 cannot be satisfied by 10. Decoding also tightened the rest: the message is matched exactly instead of by substring, total_occurrences must be present rather than merely mentioned, the gRPC code must be PermissionDenied, and the record must name a service account. Every one of these was checked by removing the behaviour it describes and confirming the test fails: an empty users page, a sampling schedule that skips occurrence 1, a 404 in place of the 403, each log field dropped in turn, a reworded message, a single walk instead of two, and the counter reset removed. Co-authored-by: c1-squire-dev[bot] --- pkg/connector/credential_lifecycle_test.go | 79 +++++++++++++++++----- 1 file changed, 62 insertions(+), 17 deletions(-) diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index d18956f8..bb333729 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -3,6 +3,7 @@ package connector import ( "bytes" "context" + "encoding/json" "fmt" "net/http" "net/http/httptest" @@ -693,9 +694,22 @@ func TestShouldLogSampled(t *testing.T) { require.False(t, shouldLogSampled(-1), "a negative count is not an occurrence") } -// drainForSkipWarnings runs one full List walk with its own log sink and -// returns the skip-warning lines that walk emitted. -func drainForSkipWarnings(t *testing.T, builder *applicationKeyBuilder) []string { +// skipWarning is one decoded skip-warning log record. +type skipWarning struct { + serviceAccountID string + code string + totalOccurrences int64 +} + +// drainForSkipWarnings runs one full List walk with its own log sink and returns +// the skip warnings that walk emitted, decoded from the JSON log records rather +// than substring-matched: total_occurrences is compared numerically, so `1` +// cannot be satisfied by `10`, `100` or `1000`, and the assertions do not rot if +// zap's encoding changes. +// +// Requires a builder whose walk visits at least one users page with data -- it +// asserts the walk did not end on its first call. +func drainForSkipWarnings(t *testing.T, builder *applicationKeyBuilder) []skipWarning { t.Helper() var logBuf bytes.Buffer core := zapcore.NewCore( @@ -706,27 +720,53 @@ func drainForSkipWarnings(t *testing.T, builder *applicationKeyBuilder) []string logger := zap.New(core) ctx := ctxzap.ToContext(context.Background(), logger) + calls := 0 token := "" - for call := 0; call < 200; call++ { - require.Less(t, call, 199, "List did not terminate") + for { + require.Less(t, calls, 200, "List did not terminate") _, results, err := builder.List(ctx, nil, rs.SyncOpAttrs{PageToken: pagination.Token{Token: token}}) require.NoError(t, err, "a 403 must never fail the sync") require.NotNil(t, results) if results.NextPageToken == "" { + // List treats an empty INCOMING token as "first call of this walk" + // and resets the sampling counter on it. That is only sound because + // a walk cannot both start and end on the same call -- otherwise an + // empty token would be ambiguous between the two. Assert it here, + // in the branch where it can actually fail; a walk that terminated + // on call zero never ran at all. + require.NotZero(t, calls, "walk ended on its first call, so an empty token is ambiguous between start and end") break } - // Only the first call of a walk may carry an empty token; the reset in - // List depends on that, so assert it rather than assuming it. - require.NotEmpty(t, results.NextPageToken, "mid-walk token must never be empty") token = results.NextPageToken + calls++ } require.NoError(t, logger.Sync()) - var skips []string + var skips []skipWarning for _, line := range strings.Split(strings.TrimSpace(logBuf.String()), "\n") { - if line != "" && strings.Contains(line, "skipping application keys for service account") { - skips = append(skips, line) + if line == "" { + continue } + var rec struct { + Msg string `json:"msg"` + ServiceAccountID string `json:"service_account_id"` + Code string `json:"code"` + TotalOccurrences *int64 `json:"total_occurrences"` + } + require.NoErrorf(t, json.Unmarshal([]byte(line), &rec), "log line is not JSON: %s", line) + // Matched exactly, not by substring, so a reworded production message + // fails the sampling assertions loudly instead of quietly matching + // nothing. (Inline rather than a named constant: gosec G101 reads a + // const holding this sentence as a hardcoded credential.) + if rec.Msg != "baton-datadog: skipping application keys for service account" { + continue + } + require.NotNilf(t, rec.TotalOccurrences, "skip warning must carry total_occurrences: %s", line) + skips = append(skips, skipWarning{ + serviceAccountID: rec.ServiceAccountID, + code: rec.Code, + totalOccurrences: *rec.TotalOccurrences, + }) } return skips } @@ -742,7 +782,8 @@ func drainForSkipWarnings(t *testing.T, builder *applicationKeyBuilder) []string // the first sync consume the 1/10/100 slots and leave every later sync silent // until the running total reached 1000 -- which is worse than the noise the // sampling exists to prevent. Draining twice proves the second walk gets its -// own schedule. +// own schedule, and the occurrence numbers are compared exactly so a later +// occurrence cannot pass as the first. func TestApplicationKeyBuilderListSamplesSkipWarning(t *testing.T) { const serviceAccounts = 12 entries := make([]string, 0, serviceAccounts) @@ -762,14 +803,18 @@ func TestApplicationKeyBuilderListSamplesSkipWarning(t *testing.T) { for walk := 1; walk <= 2; walk++ { skips := drainForSkipWarnings(t, builder) require.Lenf(t, skips, 2, "walk %d: 12 skips must log only the 1st and 10th, not one line each", walk) - for _, line := range skips { - require.Containsf(t, line, "total_occurrences", "walk %d: a sampled warning must report the real count", walk) + require.Equalf(t, int64(1), skips[0].totalOccurrences, "walk %d: first logged line must be occurrence 1, not a later one", walk) + require.Equalf(t, int64(10), skips[1].totalOccurrences, "walk %d: second logged line must be occurrence 10", walk) + for i, skip := range skips { + require.Equalf(t, codes.PermissionDenied.String(), skip.code, + "walk %d line %d: a 403 must be reported as PermissionDenied", walk, i) + require.NotEmptyf(t, skip.serviceAccountID, "walk %d line %d: skip must name the service account", walk, i) } - require.Containsf(t, skips[0], `"total_occurrences":1`, "walk %d: first logged line is occurrence 1", walk) - require.Containsf(t, skips[1], `"total_occurrences":10`, "walk %d: second logged line is occurrence 10", walk) } - // Both walks attempted every service account. + // Both walks attempted every service account. This is also what catches a + // walk that terminates early: a paging change that ended the walk before + // draining every service account would show up here as a short count. attempted := 0 for _, req := range *requests { if strings.Contains(req.path, "/application_keys") { From 4cb1157c8efb4426af44876e72cd16c1bda2d522 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Sat, 22 Aug 2026 04:56:56 +0000 Subject: [PATCH 28/49] feat: surface application-key scopes as resource profile data SecretTrait has no scopes field, so a synced application key gave C1 no way to see what the credential is actually allowed to do. Resource.Profile is free-form and has a first-class setter, so the scopes ride there, on both the sync path and the issuance path -- a freshly vended key reports its scopes immediately rather than only after the next sync rebuilds it. Datadog's scopes field is a three-state nullable list: absent, explicit null, or a list. The profile collapses that to two states, chosen so no state is misreported: - Datadog did not report scopes: the key is absent from the profile. Emitting an empty list would assert the key is unscoped, which the response never said, and that is the one error a consumer cannot detect. - Datadog reported an unscoped key, as explicit null or an empty list: the key is present and empty, positively stating that the key carries its owner's full permissions. - Datadog reported scopes: the key holds them. The value is therefore always a list when present and never null, so a consumer never needs a type switch, and absence is reserved for the one distinction that matters. The field is named "scopes" rather than a provider-prefixed name: a consumer should be able to learn what a synced credential can do without knowing which provider minted it, and this repo's existing profiles are unprefixed too. Issuance records what the provider echoed back rather than what was requested, since Datadog may normalize the list, so an issued key agrees with what the syncer will later report for it instead of drifting from it. That required the client to carry the created key's scopes, which it previously discarded. The scopes pass-through into CreateServiceAccountApplicationKey had never been exercised by any test. It is now, along with a scoped key, an unscoped key, a key whose scopes change between syncs, a provider that reports nothing, and the guarantee that setting a resource profile does not disturb the secret trait. Each new assertion was checked by removing the behaviour it describes and confirming the test fails. baton_capabilities.json is unchanged: profile data is per-resource sync output, not capability metadata. Verified by regenerating and diffing. Co-authored-by: c1-squire-dev[bot] --- pkg/client/client.go | 32 ++- pkg/connector/application_key.go | 65 +++++- pkg/connector/credential_lifecycle_test.go | 234 +++++++++++++++++++++ pkg/connector/users.go | 9 +- 4 files changed, 336 insertions(+), 4 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index f2b5506c..29c052f3 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -248,6 +248,31 @@ type IssuedApplicationKey struct { ID string Secret string ServiceAccountID string + // Scopes is what the provider echoed back for the created key, which is + // authoritative over what was requested: Datadog is free to normalize the + // list. nil means Datadog did not report scopes at all, which is distinct + // from a non-nil empty slice -- Datadog represents an unscoped key (one + // carrying its owner's full permissions) as an explicit JSON null, and + // collapsing "not reported" into "unscoped" would assert something the + // response never said. See scopesFromNullableList. + Scopes *[]string +} + +// ScopesFromNullableList reads Datadog's three-state nullable scopes list into +// two states a caller can act on. Datadog distinguishes unset (field absent +// from the response), explicit null (the documented shape for an unscoped key) +// and a list. Explicit null and a list both mean "Datadog told us the scopes", +// so they collapse to a non-nil slice -- empty for unscoped. Unset stays nil so +// callers can tell "no scope restrictions" from "the provider did not say". +func ScopesFromNullableList(raw *[]string, isSet bool) *[]string { + if !isSet { + return nil + } + granted := []string{} + if raw != nil { + granted = *raw + } + return &granted } // CreateServiceAccountApplicationKey issues a new application key scoped to @@ -272,7 +297,12 @@ func (w *DatadogClient) CreateServiceAccountApplicationKey(ctx context.Context, if appKey.Id == nil || appKey.Attributes == nil || appKey.Attributes.Key == nil || *appKey.Attributes.Key == "" { return nil, fmt.Errorf("create service account application key response omitted id or key") } - return &IssuedApplicationKey{ID: *appKey.Id, Secret: *appKey.Attributes.Key, ServiceAccountID: serviceAccountID}, nil + return &IssuedApplicationKey{ + ID: *appKey.Id, + Secret: *appKey.Attributes.Key, + ServiceAccountID: serviceAccountID, + Scopes: ScopesFromNullableList(appKey.Attributes.GetScopesOk()), + }, nil } // FindServiceAccountApplicationKeyByName returns an exact name match among a diff --git a/pkg/connector/application_key.go b/pkg/connector/application_key.go index c2fa07c6..c51a63e0 100644 --- a/pkg/connector/application_key.go +++ b/pkg/connector/application_key.go @@ -284,6 +284,57 @@ func (o *applicationKeyBuilder) listApplicationKeyPage( return ret, &resource.SyncOpResults{NextPageToken: nextPageToken}, nil } +// applicationKeyScopesProfileKey is the resource-profile field carrying a +// credential's provider-granted scopes. +// +// The name is deliberately generic rather than "datadog_scopes" or +// "application_key_scopes". Resource.Profile is free-form, this connector is +// the first V2 credential-issuance implementation, and whatever it picks +// becomes the de-facto convention for the connectors that follow. A consumer +// should be able to read one field to learn what a synced credential is +// allowed to do without knowing which provider minted it, and per-provider +// prefixes would force exactly the special-casing that defeats. It also +// matches this repo's existing profile naming, which is unprefixed +// (userResource uses first_name, login, user_id). +const applicationKeyScopesProfileKey = "scopes" + +// applicationKeyProfileOptions returns the resource options carrying an +// application key's scopes, or nothing when Datadog did not report them. +// +// The representation is chosen so a consumer can tell three provider states +// apart using only two profile states, without a type switch: +// +// - Datadog did not report scopes (nil): the profile key is ABSENT. Emitting +// an empty list here would assert the key is unscoped, which the response +// never said, and that is the one error a consumer cannot detect. +// - Datadog reported an unscoped key (explicit null, or an empty list): the +// key is present and EMPTY. That positively states "no scope restrictions +// -- this key carries its owner's full permissions", which is a fact, not +// missing data. +// - Datadog reported scopes: the key is present and holds them. +// +// So the value is always a list when present, never null. Reserving absence +// for "not reported" keeps the distinction that matters, and reserving null +// for nothing avoids a key whose type varies between resources -- Profile is +// rendered generically, and a field that is sometimes null and sometimes an +// array is a burden on every reader of it. +func applicationKeyProfileOptions(scopes *[]string) []resource.ResourceOption { + if scopes == nil { + return nil + } + // structpb rejects []string outright ("proto: invalid type: []string"), so + // the list has to be widened element by element. + values := make([]interface{}, 0, len(*scopes)) + for _, scope := range *scopes { + values = append(values, scope) + } + return []resource.ResourceOption{ + resource.WithResourceProfile(map[string]interface{}{ + applicationKeyScopesProfileKey: values, + }), + } +} + // applicationKeyResource builds the synced resource for one service-account // application key. The type is unambiguous through two structured signals a // reader (or a future requester-selection surface) can consume without @@ -296,8 +347,12 @@ func (o *applicationKeyBuilder) listApplicationKeyPage( // field this connector's delete path relies on. func applicationKeyResource(appKeyID string, serviceAccountResourceID *v2.ResourceId, attrs *datadogV2.PartialApplicationKeyAttributes) (*v2.Resource, error) { name := appKeyID - if attrs != nil && attrs.Name != nil { - name = *attrs.Name + var scopes *[]string + if attrs != nil { + if attrs.Name != nil { + name = *attrs.Name + } + scopes = client.ScopesFromNullableList(attrs.GetScopesOk()) } options := []resource.SecretTraitOption{ @@ -310,6 +365,12 @@ func applicationKeyResource(appKeyID string, serviceAccountResourceID *v2.Resour resourceOptions := []resource.ResourceOption{ resource.WithParentResourceID(serviceAccountResourceID), } + // Scopes ride on the resource profile: SecretTrait has no scopes field. + // This is safe to set alongside the secret trait because NewSecretResource + // appends WithSecretTrait after the caller's options, and + // syncSecretTraitToResource only copies a trait profile up when the + // resource has none -- so the profile set here is never clobbered. + resourceOptions = append(resourceOptions, applicationKeyProfileOptions(scopes)...) if attrs != nil && attrs.CreatedAt != nil { createdAt, err := time.Parse(time.RFC3339Nano, *attrs.CreatedAt) if err != nil { diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index bb333729..dc892196 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -11,8 +11,10 @@ import ( "testing" "github.com/DataDog/datadog-api-client-go/v2/api/datadog" + "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" "github.com/conductorone/baton-datadog/pkg/client" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" "github.com/conductorone/baton-sdk/pkg/pagination" rs "github.com/conductorone/baton-sdk/pkg/types/resource" @@ -823,3 +825,235 @@ func TestApplicationKeyBuilderListSamplesSkipWarning(t *testing.T) { } require.Equal(t, serviceAccounts*2, attempted, "every service account must be attempted on every walk") } + +// --- application-key scopes on the resource profile ----------------------- + +// profileScopes reads the scopes list out of a resource profile. The second +// return reports whether the key was present at all, which is the distinction +// the representation turns on: absent means Datadog never reported scopes, +// present-and-empty means Datadog reported an unscoped key. +func profileScopes(t *testing.T, r *v2.Resource) ([]string, bool) { + t.Helper() + profile := r.GetProfile() + if profile == nil { + return nil, false + } + value, ok := profile.GetFields()[applicationKeyScopesProfileKey] + if !ok { + return nil, false + } + list := value.GetListValue() + require.NotNil(t, list, "scopes must always be a list when present, never null") + out := make([]string, 0, len(list.GetValues())) + for _, item := range list.GetValues() { + out = append(out, item.GetStringValue()) + } + return out, true +} + +// TestApplicationKeyResourceScopesProfile: SecretTrait has no scopes field, so +// scopes ride on Resource.Profile. Datadog's scopes field is a three-state +// nullable list, and the profile must collapse it to two states without ever +// claiming a key is unscoped when the provider did not say so. Attributes are +// decoded from JSON rather than hand-built so the real wire shapes are what +// gets exercised. +func TestApplicationKeyResourceScopesProfile(t *testing.T) { + tests := []struct { + name string + attrsJSON string + wantScopes []string + wantKey bool + }{ + { + name: "scopes reported", + attrsJSON: `{"name":"k","scopes":["dashboards_read","dashboards_write"]}`, + wantScopes: []string{"dashboards_read", "dashboards_write"}, + wantKey: true, + }, + { + name: "explicit null means unscoped", + attrsJSON: `{"name":"k","scopes":null}`, + wantScopes: []string{}, + wantKey: true, + }, + { + name: "empty list also means unscoped", + attrsJSON: `{"name":"k","scopes":[]}`, + wantScopes: []string{}, + wantKey: true, + }, + { + name: "field absent means not reported, so no claim is made", + attrsJSON: `{"name":"k"}`, + wantKey: false, + }, + } + parent := &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID} + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + var attrs datadogV2.PartialApplicationKeyAttributes + require.NoError(t, json.Unmarshal([]byte(tc.attrsJSON), &attrs)) + + res, err := applicationKeyResource("appkey-1", parent, &attrs) + require.NoError(t, err) + + got, present := profileScopes(t, res) + require.Equalf(t, tc.wantKey, present, "profile key presence for %s", tc.attrsJSON) + if tc.wantKey { + require.Equal(t, tc.wantScopes, got) + } + + // Setting the profile must not cost the secret trait: + // NewSecretResource applies WithSecretTrait after the caller's + // resource options, and its trait-to-resource copy is guarded on + // the resource having no profile. + trait := &v2.SecretTrait{} + annos := annotations.Annotations(res.GetAnnotations()) + found, err := annos.Pick(trait) + require.NoError(t, err) + require.True(t, found, "secret trait must survive alongside the profile") + require.Equal(t, "datadog.service_account_application_key", trait.GetCredentialDetail()) + }) + } +} + +// TestApplicationKeyListCarriesScopes: the scopes reach the resources the sync +// actually emits, not just the constructor, and a key whose scopes change +// between syncs reports the new value rather than a cached one. +func TestApplicationKeyListCarriesScopes(t *testing.T) { + scoped := `{"id":"k-scoped","type":"application_keys","attributes":{"name":"scoped","scopes":["logs_read"]}}` + unscoped := `{"id":"k-unscoped","type":"application_keys","attributes":{"name":"unscoped","scopes":null}}` + usersPages := []string{`[{"id":"sa-1","type":"users","attributes":{"service_account":true}}]`} + + // currentKeys is swapped between drains to simulate a scope change upstream. + currentKeys := "[" + scoped + "," + unscoped + "]" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + page := 0 + if raw := r.URL.Query().Get("page[number]"); raw != "" { + _, _ = fmt.Sscanf(raw, "%d", &page) + } + switch { + case r.URL.Path == "/api/v2/users": + body := "[]" + if page < len(usersPages) { + body = usersPages[page] + } + _, _ = w.Write([]byte(`{"data":` + body + `}`)) + case strings.HasSuffix(r.URL.Path, "/application_keys"): + body := "[]" + if page == 0 { + body = currentKeys + } + _, _ = w.Write([]byte(`{"data":` + body + `}`)) + default: + t.Errorf("unexpected provider request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + } + })) + defer server.Close() + + builder := newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)) + drain := func() map[string][]string { + t.Helper() + out := map[string][]string{} + token := "" + for call := 0; call < 50; call++ { + got, results, err := builder.List(context.Background(), nil, rs.SyncOpAttrs{PageToken: pagination.Token{Token: token}}) + require.NoError(t, err) + for _, r := range got { + scopes, present := profileScopes(t, r) + require.True(t, present, "synced key %s must report scopes", r.GetId().GetResource()) + out[r.GetId().GetResource()] = scopes + } + require.NotNil(t, results) + if results.NextPageToken == "" { + return out + } + token = results.NextPageToken + } + t.Fatal("List did not terminate") + return nil + } + + first := drain() + require.Equal(t, []string{"logs_read"}, first["k-scoped"]) + require.Equal(t, []string{}, first["k-unscoped"], "an unscoped key reports an empty list, not a missing key") + + // The same key, rescoped upstream. + currentKeys = `[{"id":"k-scoped","type":"application_keys","attributes":{"name":"scoped","scopes":["logs_read","metrics_read"]}}]` + second := drain() + require.Equal(t, []string{"logs_read", "metrics_read"}, second["k-scoped"], "a rescoped key must report its new scopes") +} + +// TestIssuePassesScopesToProviderAndProfile: the requested scopes must actually +// reach Datadog's create call -- a pass-through that no test exercised before -- +// and the issued resource must carry the scopes the provider echoed back, so a +// freshly vended key agrees with what the next sync will report for it. +func TestIssuePassesScopesToProviderAndProfile(t *testing.T) { + const handle = "handle-scoped-1" + requested := []string{"logs_read", "metrics_read"} + + var createBody string + appKeysPath := "/api/v2/service_accounts/" + testServiceAccountID + "/application_keys" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/users/"+testServiceAccountID: + _, _ = w.Write([]byte(`{"data":{"id":"` + testServiceAccountID + `","type":"users","attributes":{"service_account":true}}}`)) + case r.Method == http.MethodGet && r.URL.Path == appKeysPath: + _, _ = w.Write([]byte(`{"data":[]}`)) + case r.Method == http.MethodPost && r.URL.Path == appKeysPath: + buf := new(bytes.Buffer) + _, _ = buf.ReadFrom(r.Body) + createBody = buf.String() + // Echo the scopes back, as Datadog does. + _, _ = w.Write([]byte(`{"data":{"id":"` + handle + `","type":"application_keys","attributes":{"key":"plaintext","name":"c1-req-scoped","scopes":["logs_read","metrics_read"]}}}`)) + default: + t.Errorf("unexpected provider request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + } + })) + defer server.Close() + + issuer := newCredentialUserBuilder(newLifecycleTestWrapper(server.URL)) + out, err := issuer.Issue(context.Background(), &connectorbuilder.CredentialIssueInput{ + IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID}, + RequestID: "req-scoped", + CredentialOptions: v2.CredentialIssueOptions_builder{ + ApiKey: v2.CredentialIssueOptions_ApiKey_builder{Scopes: requested}.Build(), + }.Build(), + }) + require.NoError(t, err) + + // The pass-through into the provider request. + require.NotEmpty(t, createBody, "create request body must have been captured") + var sent struct { + Data struct { + Attributes struct { + Scopes *[]string `json:"scopes"` + } `json:"attributes"` + } `json:"data"` + } + require.NoError(t, json.Unmarshal([]byte(createBody), &sent)) + require.NotNil(t, sent.Data.Attributes.Scopes, "requested scopes must be sent to Datadog") + require.Equal(t, requested, *sent.Data.Attributes.Scopes) + + // And onto the issued resource. + scopes, present := profileScopes(t, out.Secret) + require.True(t, present, "an issued key must report its scopes immediately") + require.Equal(t, requested, scopes) +} + +// TestIssueOmitsScopesProfileWhenProviderSilent: when the create response says +// nothing about scopes, the issued resource must not claim the key is unscoped. +func TestIssueOmitsScopesProfileWhenProviderSilent(t *testing.T) { + const handle = "handle-silent-1" + server, _ := newServiceAccountAppKeyServer(t, testServiceAccountID, handle, "plaintext", "c1-req-silent") + defer server.Close() + + out := issueServiceAccountAppKey(t, context.Background(), newLifecycleTestWrapper(server.URL), testServiceAccountID, "req-silent") + + _, present := profileScopes(t, out.Secret) + require.False(t, present, "a silent provider response must not be reported as an unscoped key") +} diff --git a/pkg/connector/users.go b/pkg/connector/users.go index 0743cae9..8b4d156a 100644 --- a/pkg/connector/users.go +++ b/pkg/connector/users.go @@ -87,7 +87,14 @@ func (u *credentialUserBuilder) Issue(ctx context.Context, input *connectorbuild rs.WithSecretType(v2.SecretTrait_CREDENTIAL_TYPE_STATIC_SECRET), rs.WithSecretDetail("datadog.service_account_application_key"), } - secret, err := rs.NewSecretResource(name, serviceAccountApplicationKeyResourceType, key.ID, secretTraitOptions, rs.WithParentResourceID(input.IdentityID)) + // Carry the key's scopes on the freshly issued resource so a vended key + // reports what it can do immediately, rather than only after the next sync + // rebuilds it. key.Scopes is what Datadog echoed back rather than what was + // requested, so this agrees with what the syncer will later produce for the + // same key instead of drifting from it. + resourceOptions := append([]rs.ResourceOption{rs.WithParentResourceID(input.IdentityID)}, + applicationKeyProfileOptions(key.Scopes)...) + secret, err := rs.NewSecretResource(name, serviceAccountApplicationKeyResourceType, key.ID, secretTraitOptions, resourceOptions...) if err != nil { if deleteErr := u.wrapper.DeleteServiceAccountApplicationKey(ctx, serviceAccountID, key.ID); deleteErr != nil { ctxzap.Extract(ctx).Warn("failed to clean up Datadog service account application key after resource construction error", From c8845e75fe759e520d6b1c869c796bc3fb0ebc93 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Sat, 22 Aug 2026 05:35:08 +0000 Subject: [PATCH 29/49] test: guard fake-provider state shared with handler goroutines A fake provider's handler runs on the httptest server's goroutine while the test body reads and writes the same variables. net/http promises no happens-before edge between the two, so three sites in the tests added for this PR relied on runtime implementation detail rather than on the memory model: - TestApplicationKeyListCarriesScopes rewrote the upstream key set between two drains while the handler was reading it. - TestIssuePassesScopesToProviderAndProfile captured the create request body in the handler and read it from the test body. - recordRequest appended to a shared slice from the handler, which every test using a recording fake then iterated. Guarding the helper is the minimal fix for the call sites this PR added, and it covers the pre-existing ones in the same file as a side effect. Each is now behind a mutex, with accessors at both ends. The rescope test keeps a single server across both drains on purpose: giving each drain its own server would remove the race by no longer exercising the rescope-mid-life path the test exists for. All three guards were checked by neutering them and confirming the assertions they protect fail. Worth recording for anyone auditing this later: `go test -race` does NOT report any of these, at this commit or in a minimal isolated reproduction run twenty times. The socket path between an httptest client and its handler appears to carry enough runtime synchronization for the detector to see an edge. So this is a robustness fix argued from the memory model, not a detector-confirmed failure, and -race is added to the gates because it is cheap and catches other classes -- not because it would have caught this. Co-authored-by: c1-squire-dev[bot] --- pkg/connector/credential_lifecycle_test.go | 100 ++++++++++++++++----- 1 file changed, 79 insertions(+), 21 deletions(-) diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index dc892196..c959de94 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "testing" "github.com/DataDog/datadog-api-client-go/v2/api/datadog" @@ -58,10 +59,34 @@ func recordRequest(t *testing.T, requests *[]recordedRequest, r *http.Request) r header: r.Header.Clone(), body: string(bodyBytes), } - *requests = append(*requests, rec) + appendRequest(requests, rec) return rec } +// recordMu guards the recorded-request slices. A fake provider's handler runs on +// the httptest server's goroutine while the test body reads what it recorded, +// and net/http promises no happens-before edge between the two -- relying on one +// is relying on runtime implementation detail rather than on the memory model. +// The race detector does not currently report this shape, which is why it has to +// be reasoned about rather than discovered. +var recordMu sync.Mutex + +func appendRequest(requests *[]recordedRequest, rec recordedRequest) { + recordMu.Lock() + defer recordMu.Unlock() + *requests = append(*requests, rec) +} + +// snapshotRequests copies the recorded requests under the lock so the test body +// can iterate them without racing a handler still appending. +func snapshotRequests(requests *[]recordedRequest) []recordedRequest { + recordMu.Lock() + defer recordMu.Unlock() + out := make([]recordedRequest, len(*requests)) + copy(out, *requests) + return out +} + // --- organization API key (apiTokenBuilder) coverage ----------------------- // // apiTokenBuilder / the "api-key" resource type is unchanged by the SPEC-07 @@ -115,9 +140,10 @@ func TestApiTokenBuilderDeleteUsesHandleNotSecret(t *testing.T) { require.NoError(t, err) var deleteReq *recordedRequest - for i := range *requests { - if (*requests)[i].method == http.MethodDelete { - deleteReq = &(*requests)[i] + recorded := snapshotRequests(requests) + for i := range recorded { + if recorded[i].method == http.MethodDelete { + deleteReq = &recorded[i] } } require.NotNil(t, deleteReq, "expected a DELETE request to reach the provider") @@ -397,9 +423,10 @@ func TestApplicationKeyBuilderDeleteUsesServiceAccountAPI(t *testing.T) { require.NoError(t, err) var deleteReq *recordedRequest - for i := range *requests { - if (*requests)[i].method == http.MethodDelete { - deleteReq = &(*requests)[i] + recorded := snapshotRequests(requests) + for i := range recorded { + if recorded[i].method == http.MethodDelete { + deleteReq = &recorded[i] } } require.NotNil(t, deleteReq, "expected a DELETE request to reach the provider") @@ -557,10 +584,10 @@ func drainAppKeyList(t *testing.T, builder *applicationKeyBuilder, requests *[]r const maxCalls = 100 for call := 0; ; call++ { require.Less(t, call, maxCalls, "List did not terminate") - before := len(*requests) + before := len(snapshotRequests(requests)) got, results, err := builder.List(ctx, nil, rs.SyncOpAttrs{PageToken: pagination.Token{Token: token}}) require.NoError(t, err) - requestsPerCall = append(requestsPerCall, len(*requests)-before) + requestsPerCall = append(requestsPerCall, len(snapshotRequests(requests))-before) all = append(all, got...) require.NotNil(t, results) if results.NextPageToken == "" { @@ -598,7 +625,7 @@ func TestApplicationKeyBuilderListReturnsOnePagePerCall(t *testing.T) { // separate calls is the point: draining them inside one call is what F2 // forbids, and would show up here as a single call issuing them all. require.GreaterOrEqual(t, len(requestsPerCall), 5, "the walk must span multiple List calls, one provider page each") - require.Len(t, *requests, len(requestsPerCall), "one provider request per List call") + require.Len(t, snapshotRequests(requests), len(requestsPerCall), "one provider request per List call") require.Len(t, got, defaultV2PageSize+1+2, "every application key across both service accounts must be returned") @@ -612,7 +639,7 @@ func TestApplicationKeyBuilderListReturnsOnePagePerCall(t *testing.T) { require.True(t, ids["sa2key-0"], "sa-2 keys must be present") // A human user must never be queried for service-account application keys. - for _, req := range *requests { + for _, req := range snapshotRequests(requests) { require.NotContains(t, req.path, "human-1") } } @@ -636,7 +663,7 @@ func TestApplicationKeyBuilderListSkipsForbiddenServiceAccount(t *testing.T) { require.Equal(t, "okkey-0", got[0].GetId().GetResource()) attempted := false - for _, req := range *requests { + for _, req := range snapshotRequests(requests) { if strings.Contains(req.path, "sa-forbidden") { attempted = true } @@ -818,7 +845,7 @@ func TestApplicationKeyBuilderListSamplesSkipWarning(t *testing.T) { // walk that terminates early: a paging change that ended the walk before // draining every service account would show up here as a short count. attempted := 0 - for _, req := range *requests { + for _, req := range snapshotRequests(requests) { if strings.Contains(req.path, "/application_keys") { attempted++ } @@ -925,8 +952,23 @@ func TestApplicationKeyListCarriesScopes(t *testing.T) { unscoped := `{"id":"k-unscoped","type":"application_keys","attributes":{"name":"unscoped","scopes":null}}` usersPages := []string{`[{"id":"sa-1","type":"users","attributes":{"service_account":true}}]`} - // currentKeys is swapped between drains to simulate a scope change upstream. + // The upstream key set is swapped between drains to simulate a rescope. The + // handler reads it on the server's goroutine while the test body rewrites it, + // so it is guarded: net/http gives no happens-before edge between the two. + // The same server serves both drains on purpose -- giving each drain its own + // server would stop exercising the rescope-mid-life path this test exists for. + var keysMu sync.Mutex currentKeys := "[" + scoped + "," + unscoped + "]" + setKeys := func(v string) { + keysMu.Lock() + defer keysMu.Unlock() + currentKeys = v + } + getKeys := func() string { + keysMu.Lock() + defer keysMu.Unlock() + return currentKeys + } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") page := 0 @@ -943,7 +985,7 @@ func TestApplicationKeyListCarriesScopes(t *testing.T) { case strings.HasSuffix(r.URL.Path, "/application_keys"): body := "[]" if page == 0 { - body = currentKeys + body = getKeys() } _, _ = w.Write([]byte(`{"data":` + body + `}`)) default: @@ -980,8 +1022,8 @@ func TestApplicationKeyListCarriesScopes(t *testing.T) { require.Equal(t, []string{"logs_read"}, first["k-scoped"]) require.Equal(t, []string{}, first["k-unscoped"], "an unscoped key reports an empty list, not a missing key") - // The same key, rescoped upstream. - currentKeys = `[{"id":"k-scoped","type":"application_keys","attributes":{"name":"scoped","scopes":["logs_read","metrics_read"]}}]` + // The same key, rescoped upstream, on the same live server. + setKeys(`[{"id":"k-scoped","type":"application_keys","attributes":{"name":"scoped","scopes":["logs_read","metrics_read"]}}]`) second := drain() require.Equal(t, []string{"logs_read", "metrics_read"}, second["k-scoped"], "a rescoped key must report its new scopes") } @@ -994,7 +1036,22 @@ func TestIssuePassesScopesToProviderAndProfile(t *testing.T) { const handle = "handle-scoped-1" requested := []string{"logs_read", "metrics_read"} - var createBody string + // createBody is written on the server's goroutine and read by the test body, + // so it is guarded for the same reason as the key set above. + var ( + bodyMu sync.Mutex + createBody string + ) + setCreateBody := func(v string) { + bodyMu.Lock() + defer bodyMu.Unlock() + createBody = v + } + getCreateBody := func() string { + bodyMu.Lock() + defer bodyMu.Unlock() + return createBody + } appKeysPath := "/api/v2/service_accounts/" + testServiceAccountID + "/application_keys" server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") @@ -1006,7 +1063,7 @@ func TestIssuePassesScopesToProviderAndProfile(t *testing.T) { case r.Method == http.MethodPost && r.URL.Path == appKeysPath: buf := new(bytes.Buffer) _, _ = buf.ReadFrom(r.Body) - createBody = buf.String() + setCreateBody(buf.String()) // Echo the scopes back, as Datadog does. _, _ = w.Write([]byte(`{"data":{"id":"` + handle + `","type":"application_keys","attributes":{"key":"plaintext","name":"c1-req-scoped","scopes":["logs_read","metrics_read"]}}}`)) default: @@ -1027,7 +1084,8 @@ func TestIssuePassesScopesToProviderAndProfile(t *testing.T) { require.NoError(t, err) // The pass-through into the provider request. - require.NotEmpty(t, createBody, "create request body must have been captured") + sentBody := getCreateBody() + require.NotEmpty(t, sentBody, "create request body must have been captured") var sent struct { Data struct { Attributes struct { @@ -1035,7 +1093,7 @@ func TestIssuePassesScopesToProviderAndProfile(t *testing.T) { } `json:"attributes"` } `json:"data"` } - require.NoError(t, json.Unmarshal([]byte(createBody), &sent)) + require.NoError(t, json.Unmarshal([]byte(sentBody), &sent)) require.NotNil(t, sent.Data.Attributes.Scopes, "requested scopes must be sent to Datadog") require.Equal(t, requested, *sent.Data.Attributes.Scopes) From 4836a4f04d1e465bdc174110e68e731325dc36e2 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:07:04 +0000 Subject: [PATCH 30/49] fix: request a full users page, and stop letting a bad timestamp fail a sync Three review items plus one instance of the same class found while in here. pkg/client/client_test.go shared a captured request body between the httptest handler goroutine and the test body with no synchronization -- the same shape guarded in credential_lifecycle_test.go last commit, in a file the earlier sweep reached with only one of the two directions it needed. It is now behind a mutex with accessors at both ends. The users walk in applicationKeyBuilder omitted WithPageSize. Datadog's documented default page[size] is 10, so the walk ran at a tenth of the page size apiTokenBuilder.List uses -- ten times the round-trips for the same users, in a walk that already issues one application-key request per service account it finds. It now uses the shared constant. Termination is unaffected: the users level ends on an empty page, not a short one. An unparseable created_at aborted the whole application-key sync. It now drops the field and warns, sampled with total_occurrences. The timestamp is decorative -- nothing about identifying, attributing or revoking a key depends on it, unlike the handle, the owning service account or the scopes -- so failing would trade every application key in the organization becoming invisible to C1 for one missing display value on one key. For a security product, losing sight of live credentials is the worse failure. apiTokenBuilder.List had the identical defect for created_at and modified_at on organization API keys, and aborted the walk the same way. Fixed the same way, for the same reason: deleting an org API key depends on the handle alone. That code predates this branch, but it is the sibling secret type this PR extends with delete, and the failure mode is a whole-sync outage, so it is worth carrying now rather than leaving adjacent to work under review. Every new assertion was checked by removing the behaviour it describes and confirming the test fails, including that the page-size assertion pins the value rather than merely its presence. Co-authored-by: c1-squire-dev[bot] --- pkg/client/client_test.go | 27 ++- pkg/connector/api_token.go | 43 +++- pkg/connector/application_key.go | 59 +++++- pkg/connector/credential_lifecycle_test.go | 221 ++++++++++++++++++++- 4 files changed, 335 insertions(+), 15 deletions(-) diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index d3072049..6ef20c79 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -8,6 +8,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "testing" "time" @@ -156,10 +157,27 @@ func TestServiceAccountApplicationKeyManagement(t *testing.T) { }) t.Run("create sends requested scopes", func(t *testing.T) { - var body string + // body is written on the httptest server's goroutine and read by the test + // body. net/http promises no happens-before edge between the two, so the + // mutex is load-bearing rather than decorative -- the race detector does + // not report this shape, so it has to be reasoned about. + var ( + bodyMu sync.Mutex + body string + ) + setBody := func(v string) { + bodyMu.Lock() + defer bodyMu.Unlock() + body = v + } + getBody := func() string { + bodyMu.Lock() + defer bodyMu.Unlock() + return body + } server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { buf, _ := io.ReadAll(r.Body) - body = string(buf) + setBody(string(buf)) w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`{"data":{"id":"appkey-id","type":"application_keys","attributes":{"key":"plaintext-app-key","name":"c1-request"}}}`)) })) @@ -167,8 +185,9 @@ func TestServiceAccountApplicationKeyManagement(t *testing.T) { _, err := newOfficialTestClient(server.URL).CreateServiceAccountApplicationKey(context.Background(), serviceAccountID, "c1-request", []string{"dashboards_read", "metrics_read"}) assertNoError(t, err, "create service account application key should succeed") - assertContains(t, body, "dashboards_read", "request body should include the requested scopes") - assertContains(t, body, "metrics_read", "request body should include the requested scopes") + sent := getBody() + assertContains(t, sent, "dashboards_read", "request body should include the requested scopes") + assertContains(t, sent, "metrics_read", "request body should include the requested scopes") }) t.Run("create rejects a response without plaintext material", func(t *testing.T) { diff --git a/pkg/connector/api_token.go b/pkg/connector/api_token.go index debbc50c..ae0a0047 100644 --- a/pkg/connector/api_token.go +++ b/pkg/connector/api_token.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "strings" + "sync/atomic" "time" "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" @@ -23,6 +24,11 @@ const defaultV2PageSize = 100 type apiTokenBuilder struct { resourceType *v2.ResourceType wrapper *client.DatadogClient + + // malformedTimestamps counts organization API keys in the current walk whose + // created_at or modified_at could not be parsed. Sampled (L7): a provider + // emitting a bad timestamp format would emit it for every key. + malformedTimestamps atomic.Int64 } var _ connectorbuilder.ResourceSyncerV2 = &apiTokenBuilder{} @@ -78,11 +84,32 @@ func (o *apiTokenBuilder) ResourceType(_ context.Context) *v2.ResourceType { return o.resourceType } +// warnMalformedTimestamp reports an unparseable provider timestamp, sampled so +// a bad format affecting every key does not emit one line per resource. +func (o *apiTokenBuilder) warnMalformedTimestamp(ctx context.Context, apiKeyID, field, raw string, err error) { + total := o.malformedTimestamps.Add(1) + if !shouldLogSampled(total) { + return + } + ctxzap.Extract(ctx).Warn( + "baton-datadog: organization API key timestamp could not be parsed; syncing the key without it", + zap.String("api_key_id", apiKeyID), + zap.String("field", field), + zap.String("value", raw), + zap.Int64("total_occurrences", total), + zap.Error(err), + ) +} + func (o *apiTokenBuilder) List( ctx context.Context, resourceID *v2.ResourceId, opts resource.SyncOpAttrs, ) ([]*v2.Resource, *resource.SyncOpResults, error) { + if opts.PageToken.Token == "" { + o.malformedTimestamps.Store(0) + } + bag, page, err := parsePageToken(opts.PageToken.Token, &v2.ResourceId{ResourceType: o.resourceType.Id}) if err != nil { return nil, nil, err @@ -119,19 +146,27 @@ func (o *apiTokenBuilder) List( var resourceOptions []resource.ResourceOption timeFormat := time.RFC3339Nano + // A timestamp this connector cannot parse drops that field rather than + // failing the walk. Neither created_at nor modified_at is load-bearing + // here -- identifying and deleting an organization API key depends on + // the handle alone -- so aborting would trade every API key in the + // organization becoming invisible to C1 for one missing display value. + // Same reasoning as applicationKeyResource; see its comment. if apiToken.Attributes != nil && apiToken.Attributes.CreatedAt != nil { createdAt, err := time.Parse(timeFormat, *apiToken.Attributes.CreatedAt) if err != nil { - return nil, nil, err + o.warnMalformedTimestamp(ctx, *apiToken.Id, "created_at", *apiToken.Attributes.CreatedAt, err) + } else { + resourceOptions = append(resourceOptions, resource.WithResourceCreatedAt(createdAt)) } - resourceOptions = append(resourceOptions, resource.WithResourceCreatedAt(createdAt)) } if apiToken.Attributes != nil && apiToken.Attributes.ModifiedAt != nil { modifiedAt, err := time.Parse(timeFormat, *apiToken.Attributes.ModifiedAt) if err != nil { - return nil, nil, err + o.warnMalformedTimestamp(ctx, *apiToken.Id, "modified_at", *apiToken.Attributes.ModifiedAt, err) + } else { + options = append(options, resource.WithSecretLastUsedAt(modifiedAt)) } - options = append(options, resource.WithSecretLastUsedAt(modifiedAt)) } name := *apiToken.Id if apiToken.Attributes != nil && apiToken.Attributes.Name != nil { diff --git a/pkg/connector/application_key.go b/pkg/connector/application_key.go index c51a63e0..273cc253 100644 --- a/pkg/connector/application_key.go +++ b/pkg/connector/application_key.go @@ -31,6 +31,12 @@ type applicationKeyBuilder struct { // service account. It is reset at the start of each walk (see List) because // this builder outlives any single sync. skippedServiceAccounts atomic.Int64 + + // malformedCreatedAt counts application keys in the current walk whose + // created_at could not be parsed. Sampled for the same reason as the skip + // warning (L7): a provider emitting a bad timestamp format would emit it for + // every key, so an unsampled warning would be one line per resource. + malformedCreatedAt atomic.Int64 } var _ connectorbuilder.ResourceSyncerV2 = &applicationKeyBuilder{} @@ -150,6 +156,7 @@ func (o *applicationKeyBuilder) List( // retried first page resets again, which is what a restarted walk wants. if opts.PageToken.Token == "" { o.skippedServiceAccounts.Store(0) + o.malformedCreatedAt.Store(0) } bag, page, err := parsePageToken(opts.PageToken.Token, &v2.ResourceId{ResourceType: o.resourceType.Id}) @@ -174,7 +181,13 @@ func (o *applicationKeyBuilder) listServiceAccountsPage( bag *pagination.Bag, page int64, ) ([]*v2.Resource, *resource.SyncOpResults, error) { - users, err := o.wrapper.ListUsers(ctx, datadogV2.NewListUsersOptionalParameters().WithPageNumber(page)) + // Datadog's documented default page[size] is 10, so omitting it would run + // this walk at a tenth of the page size apiTokenBuilder.List uses -- ten + // times the round-trips for the same users, in a walk that already fans out + // one application-key request per service account found. Termination is + // unaffected: the users level ends on an empty page, not a short one. + users, err := o.wrapper.ListUsers(ctx, + datadogV2.NewListUsersOptionalParameters().WithPageNumber(page).WithPageSize(defaultV2PageSize)) if err != nil { return nil, nil, fmt.Errorf("baton-datadog: list users while syncing service account application keys: %w", err) } @@ -263,7 +276,19 @@ func (o *applicationKeyBuilder) listApplicationKeyPage( if key.Id == nil { continue } - rv, err := applicationKeyResource(*key.Id, serviceAccountResourceID, key.Attributes) + rv, err := applicationKeyResource(*key.Id, serviceAccountResourceID, key.Attributes, + func(appKeyID string, raw string, parseErr error) { + if total := o.malformedCreatedAt.Add(1); shouldLogSampled(total) { + ctxzap.Extract(ctx).Warn( + "baton-datadog: application key created_at could not be parsed; syncing the key without it", + zap.String("application_key_id", appKeyID), + zap.String("service_account_id", serviceAccountID), + zap.String("created_at", raw), + zap.Int64("total_occurrences", total), + zap.Error(parseErr), + ) + } + }) if err != nil { return nil, nil, err } @@ -345,7 +370,16 @@ func applicationKeyProfileOptions(scopes *[]string) []resource.ResourceOption { // display-name text. WithParentResourceID records the owning service account // as the resource's parent; see Delete's doc comment for why that is the // field this connector's delete path relies on. -func applicationKeyResource(appKeyID string, serviceAccountResourceID *v2.ResourceId, attrs *datadogV2.PartialApplicationKeyAttributes) (*v2.Resource, error) { +// onMalformedCreatedAt is called when Datadog's created_at cannot be parsed. It +// may be nil, in which case the field is dropped silently. +type onMalformedCreatedAt func(appKeyID string, raw string, err error) + +func applicationKeyResource( + appKeyID string, + serviceAccountResourceID *v2.ResourceId, + attrs *datadogV2.PartialApplicationKeyAttributes, + reportMalformedCreatedAt onMalformedCreatedAt, +) (*v2.Resource, error) { name := appKeyID var scopes *[]string if attrs != nil { @@ -371,12 +405,25 @@ func applicationKeyResource(appKeyID string, serviceAccountResourceID *v2.Resour // syncSecretTraitToResource only copies a trait profile up when the // resource has none -- so the profile set here is never clobbered. resourceOptions = append(resourceOptions, applicationKeyProfileOptions(scopes)...) + // A created_at this connector cannot parse drops the field; it does not fail + // the resource, and it must not fail the walk. The timestamp is decorative + // here -- nothing about identifying, attributing or revoking the key depends + // on it, unlike the handle, the owning service account or the scopes -- so + // aborting would trade every application key in the organization becoming + // invisible to C1 for one missing display value on one key. For a security + // product, losing sight of live credentials is the worse failure. The + // provider is still misbehaving, so the caller is told and warns about it + // (sampled, since a bad format would affect every key). if attrs != nil && attrs.CreatedAt != nil { createdAt, err := time.Parse(time.RFC3339Nano, *attrs.CreatedAt) - if err != nil { - return nil, fmt.Errorf("baton-datadog: parse application key created_at: %w", err) + switch { + case err != nil: + if reportMalformedCreatedAt != nil { + reportMalformedCreatedAt(appKeyID, *attrs.CreatedAt, err) + } + default: + resourceOptions = append(resourceOptions, resource.WithResourceCreatedAt(createdAt)) } - resourceOptions = append(resourceOptions, resource.WithResourceCreatedAt(createdAt)) } return resource.NewSecretResource( diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index c959de94..970ebd01 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -921,7 +921,7 @@ func TestApplicationKeyResourceScopesProfile(t *testing.T) { var attrs datadogV2.PartialApplicationKeyAttributes require.NoError(t, json.Unmarshal([]byte(tc.attrsJSON), &attrs)) - res, err := applicationKeyResource("appkey-1", parent, &attrs) + res, err := applicationKeyResource("appkey-1", parent, &attrs, nil) require.NoError(t, err) got, present := profileScopes(t, res) @@ -1115,3 +1115,222 @@ func TestIssueOmitsScopesProfileWhenProviderSilent(t *testing.T) { _, present := profileScopes(t, out.Secret) require.False(t, present, "a silent provider response must not be reported as an unscoped key") } + +// --- malformed created_at, and users page size --------------------------- + +// TestApplicationKeyResourceMalformedCreatedAt: a created_at this connector +// cannot parse must drop the field and still produce a usable resource, and it +// must report the problem to its caller. Failing here would mean one bad +// timestamp from the provider hides every application key in the organization +// from C1, which is a worse outcome than a missing display value. +func TestApplicationKeyResourceMalformedCreatedAt(t *testing.T) { + parent := &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID} + + t.Run("malformed value drops the field and reports", func(t *testing.T) { + var attrs datadogV2.PartialApplicationKeyAttributes + require.NoError(t, json.Unmarshal([]byte(`{"name":"k","created_at":"not-a-timestamp"}`), &attrs)) + + var gotID, gotRaw string + var gotErr error + res, err := applicationKeyResource("appkey-bad-ts", parent, &attrs, + func(appKeyID, raw string, parseErr error) { + gotID, gotRaw, gotErr = appKeyID, raw, parseErr + }) + + require.NoError(t, err, "a malformed created_at must not fail resource construction") + require.NotNil(t, res) + require.Equal(t, "appkey-bad-ts", res.GetId().GetResource(), "the key must still be syncable") + require.Nil(t, res.GetCreatedAt(), "an unparseable created_at must be dropped, not guessed") + + require.Equal(t, "appkey-bad-ts", gotID, "the report must name the key") + require.Equal(t, "not-a-timestamp", gotRaw, "the report must carry the value that failed") + require.Error(t, gotErr, "the report must carry the parse error") + }) + + t.Run("a parseable value is still recorded", func(t *testing.T) { + var attrs datadogV2.PartialApplicationKeyAttributes + require.NoError(t, json.Unmarshal([]byte(`{"name":"k","created_at":"2026-08-22T04:01:01.5Z"}`), &attrs)) + + called := false + res, err := applicationKeyResource("appkey-good-ts", parent, &attrs, + func(string, string, error) { called = true }) + + require.NoError(t, err) + require.False(t, called, "a parseable created_at must not be reported as malformed") + require.NotNil(t, res.GetCreatedAt(), "a parseable created_at must be recorded") + require.Equal(t, 2026, res.GetCreatedAt().AsTime().Year()) + }) + + t.Run("a nil callback is tolerated", func(t *testing.T) { + var attrs datadogV2.PartialApplicationKeyAttributes + require.NoError(t, json.Unmarshal([]byte(`{"name":"k","created_at":"nope"}`), &attrs)) + res, err := applicationKeyResource("appkey-nil-cb", parent, &attrs, nil) + require.NoError(t, err) + require.Nil(t, res.GetCreatedAt()) + }) +} + +// TestApplicationKeyListSurvivesMalformedCreatedAt: the whole walk must complete +// when the provider returns an unparseable created_at, the affected key must +// still be synced, and the warning must be sampled with total_occurrences rather +// than emitted once per key (L7). +func TestApplicationKeyListSurvivesMalformedCreatedAt(t *testing.T) { + const keys = 12 + entries := make([]string, 0, keys) + for i := 0; i < keys; i++ { + entries = append(entries, fmt.Sprintf( + `{"id":"bad-ts-%02d","type":"application_keys","attributes":{"name":"k%02d","created_at":"not-a-timestamp"}}`, i, i)) + } + usersPages := []string{`[{"id":"sa-1","type":"users","attributes":{"service_account":true}}]`} + appKeyPages := map[string][]string{"sa-1": {"[" + strings.Join(entries, ",") + "]"}} + + server, _ := newAppKeyListServer(t, usersPages, appKeyPages, nil) + defer server.Close() + + var logBuf bytes.Buffer + core := zapcore.NewCore( + zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(&logBuf), + zapcore.DebugLevel, + ) + ctx := ctxzap.ToContext(context.Background(), zap.New(core)) + + builder := newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)) + var synced []*v2.Resource + token := "" + for call := 0; call < 50; call++ { + got, results, err := builder.List(ctx, nil, rs.SyncOpAttrs{PageToken: pagination.Token{Token: token}}) + require.NoError(t, err, "a malformed created_at must never fail the walk") + synced = append(synced, got...) + require.NotNil(t, results) + if results.NextPageToken == "" { + break + } + token = results.NextPageToken + } + + require.Len(t, synced, keys, "every key must still be synced despite the bad timestamp") + for _, r := range synced { + require.Nil(t, r.GetCreatedAt(), "the unparseable timestamp must be dropped, not guessed") + } + + warned := 0 + for _, line := range strings.Split(strings.TrimSpace(logBuf.String()), "\n") { + if line == "" { + continue + } + var rec struct { + Msg string `json:"msg"` + TotalOccurrences *int64 `json:"total_occurrences"` + } + require.NoError(t, json.Unmarshal([]byte(line), &rec)) + if rec.Msg != "baton-datadog: application key created_at could not be parsed; syncing the key without it" { + continue + } + warned++ + require.NotNil(t, rec.TotalOccurrences, "the warning must carry total_occurrences") + } + require.Equal(t, 2, warned, "12 malformed timestamps must log only the 1st and 10th, not one line each") +} + +// TestApplicationKeySyncRequestsFullUsersPage: the users walk must ask for the +// same page size the rest of the connector uses. Datadog's documented default is +// 10, so omitting it costs ten times the round-trips for the same users. +func TestApplicationKeySyncRequestsFullUsersPage(t *testing.T) { + usersPages := []string{`[{"id":"sa-1","type":"users","attributes":{"service_account":true}}]`} + appKeyPages := map[string][]string{"sa-1": {appKeyPageJSON("k", 1)}} + server, requests := newAppKeyListServer(t, usersPages, appKeyPages, nil) + defer server.Close() + + drainAppKeyList(t, newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)), requests) + + sawUsersPage := false + for _, req := range snapshotRequests(requests) { + if req.path != "/api/v2/users" { + continue + } + sawUsersPage = true + require.Containsf(t, req.query, fmt.Sprintf("page%%5Bsize%%5D=%d", defaultV2PageSize), + "users walk must request the shared page size; query was %q", req.query) + } + require.True(t, sawUsersPage, "the walk must have listed users at least once") +} + +// TestApiTokenListSurvivesMalformedTimestamps: the organization API-key sync +// must not abort because a provider timestamp will not parse. This mirrors the +// application-key behavior; deleting an org API key depends on the handle alone, +// so a bad created_at or modified_at is not worth a whole-sync outage. +func TestApiTokenListSurvivesMalformedTimestamps(t *testing.T) { + const keys = 12 + entries := make([]string, 0, keys) + for i := 0; i < keys; i++ { + entries = append(entries, fmt.Sprintf( + `{"id":"key-%02d","type":"api_keys","attributes":{"name":"n%02d","created_at":"not-a-timestamp","modified_at":"also-bad"}}`, i, i)) + } + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + page := 0 + if raw := r.URL.Query().Get("page[number]"); raw != "" { + _, _ = fmt.Sscanf(raw, "%d", &page) + } + if r.URL.Path != "/api/v2/api_keys" { + t.Errorf("unexpected provider request: %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + return + } + if page > 0 { + _, _ = w.Write([]byte(`{"data":[]}`)) + return + } + _, _ = w.Write([]byte(`{"data":[` + strings.Join(entries, ",") + `]}`)) + })) + defer server.Close() + + var logBuf bytes.Buffer + core := zapcore.NewCore( + zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), + zapcore.AddSync(&logBuf), + zapcore.DebugLevel, + ) + ctx := ctxzap.ToContext(context.Background(), zap.New(core)) + + builder := newApiTokenBuilder(newLifecycleTestWrapper(server.URL)) + var synced []*v2.Resource + token := "" + for call := 0; call < 50; call++ { + got, results, err := builder.List(ctx, nil, rs.SyncOpAttrs{PageToken: pagination.Token{Token: token}}) + require.NoError(t, err, "a malformed timestamp must never fail the org API-key walk") + synced = append(synced, got...) + require.NotNil(t, results) + if results.NextPageToken == "" { + break + } + token = results.NextPageToken + } + + require.Len(t, synced, keys, "every org API key must still be synced despite bad timestamps") + for _, r := range synced { + require.Nil(t, r.GetCreatedAt(), "an unparseable created_at must be dropped, not guessed") + } + + warned := 0 + for _, line := range strings.Split(strings.TrimSpace(logBuf.String()), "\n") { + if line == "" { + continue + } + var rec struct { + Msg string `json:"msg"` + Field string `json:"field"` + TotalOccurrences *int64 `json:"total_occurrences"` + } + require.NoError(t, json.Unmarshal([]byte(line), &rec)) + if rec.Msg != "baton-datadog: organization API key timestamp could not be parsed; syncing the key without it" { + continue + } + warned++ + require.NotNil(t, rec.TotalOccurrences, "the warning must carry total_occurrences") + require.Contains(t, []string{"created_at", "modified_at"}, rec.Field, "the warning must name the field") + } + // 12 keys x 2 bad fields = 24 occurrences, so only the 1st and 10th log. + require.Equal(t, 2, warned, "24 malformed timestamps must log only the 1st and 10th, not one line each") +} From d993c2a214516d9ba487433c9b3790326a20002c Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Mon, 24 Aug 2026 16:16:21 +0000 Subject: [PATCH 31/49] docs: reattach the applicationKeyResource doc comment to its function Adding the onMalformedCreatedAt callback type placed its declaration between the applicationKeyResource doc block and the function, so godoc read the whole block as documenting the type: applicationKeyResource was left undocumented and the type carried a comment about something else. The type now sits above the doc block with a one-line comment of its own stating what it is told and when, since a callback's contract is not obvious from its name. The parameter's nil-tolerance moves onto the function, where the parameter is. Comment-only: the sole non-comment change is the type declaration moving unmodified. Co-authored-by: c1-squire-dev[bot] --- pkg/connector/application_key.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/pkg/connector/application_key.go b/pkg/connector/application_key.go index 273cc253..72048451 100644 --- a/pkg/connector/application_key.go +++ b/pkg/connector/application_key.go @@ -360,6 +360,12 @@ func applicationKeyProfileOptions(scopes *[]string) []resource.ResourceOption { } } +// onMalformedCreatedAt is told that an application key's Datadog created_at +// could not be parsed: appKeyID identifies the key, raw is the value that +// failed, and err is the parse error. The field is dropped either way -- this +// callback only decides whether anyone hears about it. +type onMalformedCreatedAt func(appKeyID string, raw string, err error) + // applicationKeyResource builds the synced resource for one service-account // application key. The type is unambiguous through two structured signals a // reader (or a future requester-selection surface) can consume without @@ -370,10 +376,9 @@ func applicationKeyProfileOptions(scopes *[]string) []resource.ResourceOption { // display-name text. WithParentResourceID records the owning service account // as the resource's parent; see Delete's doc comment for why that is the // field this connector's delete path relies on. -// onMalformedCreatedAt is called when Datadog's created_at cannot be parsed. It -// may be nil, in which case the field is dropped silently. -type onMalformedCreatedAt func(appKeyID string, raw string, err error) - +// +// reportMalformedCreatedAt may be nil, in which case an unparseable created_at +// is dropped silently. func applicationKeyResource( appKeyID string, serviceAccountResourceID *v2.ResourceId, From f94798ce82031ca62d5d48152a3e669760491893 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Wed, 26 Aug 2026 03:00:24 +0000 Subject: [PATCH 32/49] fix: fail the sync when service-account application keys cannot be read A 403 or 404 from ListServiceAccountApplicationKeys used to warn and skip that service account, letting the sync complete without its application keys. C1 reads a resource missing from a completed sync as deleted, so an install whose Datadog role lacks service_account_write would report a healthy sync and retire its live application keys from the inventory instead of surfacing the permission problem. For a credential inventory, asserting "no keys here" when the truth is "I could not look" is the worse failure. The platform's data-drop guard does not reliably cover this. Its per-resource-type check is feature-flagged, it only engages once the previous count exceeds 25, and it skips a resource type whose previous count was zero -- which is every install that never had the permission, since this resource type is new. The skip was also per page, not per service account: it popped the pagination bag exactly as the short-page path does, so a 403 or 404 arriving on page three truncated that service account's key list mid-enumeration and reported it as fully enumerated. A partial drop is less likely to cross the guard's threshold than a total one, so that case was the one most likely to delete keys silently. Both codes fail hard now. Datadog documents 403 and 404 on this endpoint without saying which failures produce which, so a 404 cannot be assumed to mean the service account is genuinely gone rather than invisible to this role; splitting them would reproduce the same silent-emptying under a masked 403. The error names the required permission, because nothing else does -- C1 does not consume a connector's advertised CapabilityPermissions. Drop the log-sampling machinery with it. shouldLogSampled and the three atomic.Int64 counters existed to keep the skip warning from firing once per service account; with no skip there is nothing to throttle. The remaining malformed-timestamp warnings now log every occurrence, which is what every other baton connector does -- baton-sdk sets zc.Sampling = nil, so nothing upstream was sampling them either. Co-authored-by: c1-squire-dev[bot] --- docs/connector.mdx | 2 +- pkg/connector/api_token.go | 19 +- pkg/connector/application_key.go | 95 ++------ pkg/connector/credential_lifecycle_test.go | 250 +++++++-------------- pkg/connector/helpers.go | 21 -- 5 files changed, 98 insertions(+), 289 deletions(-) diff --git a/docs/connector.mdx b/docs/connector.mdx index 500f1393..196eb233 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -52,7 +52,7 @@ Configuring the connector requires you to pass in credentials generated in Datad A user with the **Connector Administrator** or **Super Administrator** role in C1 and the **Datadog Admin** or **Datadog standard** role in Datadog must perform this task. -If your user has a custom Datadog role, make sure it includes **User App Keys**, **User Access Invite**, and **User Access Manage** to create, update, enable, and disable users from C1. If you enable **Sync secrets**, also add **Service Account Write**, which governs syncing, issuing, and revoking service account application keys, plus **API Keys Read** and **API Keys Delete** to sync and revoke organization API keys. +If your user has a custom Datadog role, make sure it includes **User App Keys**, **User Access Invite**, and **User Access Manage** to create, update, enable, and disable users from C1. If you enable **Sync secrets**, also add **Service Account Write**, which governs syncing, issuing, and revoking service account application keys, plus **API Keys Read** and **API Keys Delete** to sync and revoke organization API keys. **Service Account Write** is required, not optional: with **Sync secrets** enabled, a role that lacks it fails the sync rather than syncing an application-key inventory that is silently missing keys. ### Locate your Datadog site diff --git a/pkg/connector/api_token.go b/pkg/connector/api_token.go index ae0a0047..8d44a326 100644 --- a/pkg/connector/api_token.go +++ b/pkg/connector/api_token.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strings" - "sync/atomic" "time" "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" @@ -24,11 +23,6 @@ const defaultV2PageSize = 100 type apiTokenBuilder struct { resourceType *v2.ResourceType wrapper *client.DatadogClient - - // malformedTimestamps counts organization API keys in the current walk whose - // created_at or modified_at could not be parsed. Sampled (L7): a provider - // emitting a bad timestamp format would emit it for every key. - malformedTimestamps atomic.Int64 } var _ connectorbuilder.ResourceSyncerV2 = &apiTokenBuilder{} @@ -84,19 +78,14 @@ func (o *apiTokenBuilder) ResourceType(_ context.Context) *v2.ResourceType { return o.resourceType } -// warnMalformedTimestamp reports an unparseable provider timestamp, sampled so -// a bad format affecting every key does not emit one line per resource. +// warnMalformedTimestamp reports an unparseable provider timestamp. The field +// is dropped and the key still syncs. func (o *apiTokenBuilder) warnMalformedTimestamp(ctx context.Context, apiKeyID, field, raw string, err error) { - total := o.malformedTimestamps.Add(1) - if !shouldLogSampled(total) { - return - } ctxzap.Extract(ctx).Warn( "baton-datadog: organization API key timestamp could not be parsed; syncing the key without it", zap.String("api_key_id", apiKeyID), zap.String("field", field), zap.String("value", raw), - zap.Int64("total_occurrences", total), zap.Error(err), ) } @@ -106,10 +95,6 @@ func (o *apiTokenBuilder) List( resourceID *v2.ResourceId, opts resource.SyncOpAttrs, ) ([]*v2.Resource, *resource.SyncOpResults, error) { - if opts.PageToken.Token == "" { - o.malformedTimestamps.Store(0) - } - bag, page, err := parsePageToken(opts.PageToken.Token, &v2.ResourceId{ResourceType: o.resourceType.Id}) if err != nil { return nil, nil, err diff --git a/pkg/connector/application_key.go b/pkg/connector/application_key.go index 72048451..38a8064b 100644 --- a/pkg/connector/application_key.go +++ b/pkg/connector/application_key.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "strconv" - "sync/atomic" "time" "github.com/DataDog/datadog-api-client-go/v2/api/datadogV2" @@ -23,20 +22,6 @@ import ( type applicationKeyBuilder struct { resourceType *v2.ResourceType wrapper *client.DatadogClient - - // skippedServiceAccounts counts how many service accounts the current walk - // has skipped because their application keys could not be read. The warning - // that reports a skip is sampled (L7): the case it exists for is a role - // missing service_account_write org-wide, which makes it fire for every - // service account. It is reset at the start of each walk (see List) because - // this builder outlives any single sync. - skippedServiceAccounts atomic.Int64 - - // malformedCreatedAt counts application keys in the current walk whose - // created_at could not be parsed. Sampled for the same reason as the skip - // warning (L7): a provider emitting a bad timestamp format would emit it for - // every key, so an unsampled warning would be one line per resource. - malformedCreatedAt atomic.Int64 } var _ connectorbuilder.ResourceSyncerV2 = &applicationKeyBuilder{} @@ -137,28 +122,6 @@ func (o *applicationKeyBuilder) List( _ *v2.ResourceId, opts resource.SyncOpAttrs, ) ([]*v2.Resource, *resource.SyncOpResults, error) { - // An empty page token is the first call of a walk, so the skip-warning - // sampling counter restarts here and each sync gets its own 1/10/100 - // schedule. The builder is constructed once per connector process -- - // Datadog.ResourceSyncers runs inside connectorbuilder.NewConnector, whose - // result is reused for every sync -- so without this reset the first sync - // consumes the early log slots and a later sync against the same org-wide - // missing permission would emit nothing until the running total reached - // 1000. - // - // An empty token is a safe first-walk signal for this builder: List only - // ever returns an empty NextPageToken from bag.Marshal() with an empty - // bag, and the bag can only be emptied by popping the users-level state, - // which happens solely on an empty users page -- the end of the walk. So - // no mid-walk call can carry one. (parsePageToken also accepts a "page:N" - // seed form, which nothing produces for this resource type; if one ever - // did, the counter would merely carry over rather than misbehave.) A - // retried first page resets again, which is what a restarted walk wants. - if opts.PageToken.Token == "" { - o.skippedServiceAccounts.Store(0) - o.malformedCreatedAt.Store(0) - } - bag, page, err := parsePageToken(opts.PageToken.Token, &v2.ResourceId{ResourceType: o.resourceType.Id}) if err != nil { return nil, nil, err @@ -236,35 +199,21 @@ func (o *applicationKeyBuilder) listApplicationKeyPage( maxApplicationKeyPages, serviceAccountID) } + // Every error fails the sync, including PermissionDenied and NotFound. + // Skipping the service account instead would report a successful sync that + // omits its application keys, and C1 reads a resource missing from a + // completed sync as deleted -- so a role lacking service_account_write + // would silently retire live credentials from the inventory rather than + // saying it could not read them. Datadog also documents 403 and 404 on + // this endpoint without saying which failures produce which, so a 404 here + // cannot be assumed to mean the service account is genuinely gone. resp, err := o.wrapper.ListServiceAccountApplicationKeys(ctx, serviceAccountID, page, defaultV2PageSize) if err != nil { - // ListServiceAccountApplicationKeys requires Datadog's - // service_account_write permission, which this sync path is the - // first to need: an install that already had sync-secrets on for - // read-only key inventory may run a read-mostly custom role that - // lacks it. A service account can also be deleted mid-sync. Warn and - // skip that one service account rather than failing the whole sync - // (criteria R7); every other provider error still fails hard. if code := status.Code(err); code == codes.PermissionDenied || code == codes.NotFound { - // Sampled, not per-service-account: an org-wide missing - // service_account_write would otherwise emit one line per service - // account on every sync. total_occurrences keeps the real count - // visible on the lines that do get through. - if total := o.skippedServiceAccounts.Add(1); shouldLogSampled(total) { - ctxzap.Extract(ctx).Warn( - "baton-datadog: skipping application keys for service account", - zap.String("service_account_id", serviceAccountID), - zap.String("code", code.String()), - zap.Int64("total_occurrences", total), - zap.Error(err), - ) - } - bag.Pop() - nextPageToken, marshalErr := bag.Marshal() - if marshalErr != nil { - return nil, nil, fmt.Errorf("baton-datadog: marshal pagination bag: %w", marshalErr) - } - return nil, &resource.SyncOpResults{NextPageToken: nextPageToken}, nil + return nil, nil, fmt.Errorf( + "baton-datadog: list application keys for service account %q: %w "+ + "(listing service-account application keys requires the Datadog service_account_write permission)", + serviceAccountID, err) } return nil, nil, fmt.Errorf("baton-datadog: list application keys for service account %q: %w", serviceAccountID, err) } @@ -278,16 +227,13 @@ func (o *applicationKeyBuilder) listApplicationKeyPage( } rv, err := applicationKeyResource(*key.Id, serviceAccountResourceID, key.Attributes, func(appKeyID string, raw string, parseErr error) { - if total := o.malformedCreatedAt.Add(1); shouldLogSampled(total) { - ctxzap.Extract(ctx).Warn( - "baton-datadog: application key created_at could not be parsed; syncing the key without it", - zap.String("application_key_id", appKeyID), - zap.String("service_account_id", serviceAccountID), - zap.String("created_at", raw), - zap.Int64("total_occurrences", total), - zap.Error(parseErr), - ) - } + ctxzap.Extract(ctx).Warn( + "baton-datadog: application key created_at could not be parsed; syncing the key without it", + zap.String("application_key_id", appKeyID), + zap.String("service_account_id", serviceAccountID), + zap.String("created_at", raw), + zap.Error(parseErr), + ) }) if err != nil { return nil, nil, err @@ -417,8 +363,7 @@ func applicationKeyResource( // aborting would trade every application key in the organization becoming // invisible to C1 for one missing display value on one key. For a security // product, losing sight of live credentials is the worse failure. The - // provider is still misbehaving, so the caller is told and warns about it - // (sampled, since a bad format would affect every key). + // provider is still misbehaving, so the caller is told and warns about it. if attrs != nil && attrs.CreatedAt != nil { createdAt, err := time.Parse(time.RFC3339Nano, *attrs.CreatedAt) switch { diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index 970ebd01..8acde9a8 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -644,11 +644,16 @@ func TestApplicationKeyBuilderListReturnsOnePagePerCall(t *testing.T) { } } -// TestApplicationKeyBuilderListSkipsForbiddenServiceAccount: a 403 from -// ListServiceAccountApplicationKeys -- what an install whose Datadog role -// lacks service_account_write gets -- must skip that one service account and -// let the rest of the sync finish, not fail the whole sync (criteria R7). -func TestApplicationKeyBuilderListSkipsForbiddenServiceAccount(t *testing.T) { +// TestApplicationKeyBuilderListFailsHardOnForbiddenServiceAccount: a 403 from +// ListServiceAccountApplicationKeys -- what an install whose Datadog role lacks +// service_account_write gets -- must fail the sync. Skipping the service +// account would report a successful sync missing its application keys, and C1 +// reads a resource absent from a completed sync as deleted, so the keys would +// be retired from the inventory instead of the permission problem surfacing. +// The error has to name the permission, because nothing else does: C1 does not +// consume a connector's advertised CapabilityPermissions, so this message is +// the only place an operator learns what to grant. +func TestApplicationKeyBuilderListFailsHardOnForbiddenServiceAccount(t *testing.T) { usersPages := []string{ `[{"id":"sa-forbidden","type":"users","attributes":{"service_account":true}},` + `{"id":"sa-ok","type":"users","attributes":{"service_account":true}}]`, @@ -657,18 +662,66 @@ func TestApplicationKeyBuilderListSkipsForbiddenServiceAccount(t *testing.T) { server, requests := newAppKeyListServer(t, usersPages, appKeyPages, map[string]bool{"sa-forbidden": true}) defer server.Close() - got, _ := drainAppKeyList(t, newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)), requests) + builder := newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)) + ctx := context.Background() + token := "" + for call := 0; call < 10; call++ { + _, results, err := builder.List(ctx, nil, rs.SyncOpAttrs{PageToken: pagination.Token{Token: token}}) + if err != nil { + require.ErrorContains(t, err, "service_account_write", + "the error must name the permission the operator has to grant") + require.ErrorContains(t, err, "sa-forbidden", + "the error must name the service account that could not be read") + attempted := false + for _, req := range snapshotRequests(requests) { + if strings.Contains(req.path, "sa-forbidden") { + attempted = true + } + } + require.True(t, attempted, "the forbidden service account must actually have been attempted") + return + } + require.NotNil(t, results) + require.NotEmpty(t, results.NextPageToken, "sync ended without surfacing the 403") + token = results.NextPageToken + } + t.Fatal("List never surfaced the 403") +} - require.Len(t, got, 1, "the readable service account's keys must still sync") - require.Equal(t, "okkey-0", got[0].GetId().GetResource()) +// TestApplicationKeyBuilderListFailsHardOnMissingServiceAccount: a 404 is +// treated exactly like a 403. Datadog documents both on this endpoint without +// saying which failures produce which, so a 404 cannot be assumed to mean the +// service account is genuinely gone rather than invisible to this role -- +// and a role masked behind 404s would otherwise sync an empty credential +// inventory while reporting success. +func TestApplicationKeyBuilderListFailsHardOnMissingServiceAccount(t *testing.T) { + requests := &[]recordedRequest{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + recordRequest(t, requests, r) + w.Header().Set("Content-Type", "application/json") + if r.URL.Path == "/api/v2/users" { + _, _ = w.Write([]byte(`{"data":[{"id":"sa-gone","type":"users","attributes":{"service_account":true}}]}`)) + return + } + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":["Not Found"]}`)) + })) + defer server.Close() - attempted := false - for _, req := range snapshotRequests(requests) { - if strings.Contains(req.path, "sa-forbidden") { - attempted = true + builder := newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)) + ctx := context.Background() + token := "" + for call := 0; call < 10; call++ { + _, results, err := builder.List(ctx, nil, rs.SyncOpAttrs{PageToken: pagination.Token{Token: token}}) + if err != nil { + require.ErrorContains(t, err, "sa-gone") + return } + require.NotNil(t, results) + require.NotEmpty(t, results.NextPageToken, "sync ended without surfacing the 404") + token = results.NextPageToken } - require.True(t, attempted, "the forbidden service account must actually have been attempted") + t.Fatal("List never surfaced the 404") } // TestApplicationKeyBuilderListFailsHardOnUnexpectedError: only @@ -703,156 +756,6 @@ func TestApplicationKeyBuilderListFailsHardOnUnexpectedError(t *testing.T) { t.Fatal("List never surfaced the provider 5xx") } -// TestShouldLogSampled: the L7 schedule is the 1st, 10th and 100th occurrence, -// then every 1000th, and nothing else. -func TestShouldLogSampled(t *testing.T) { - logged := map[int64]bool{} - for n := int64(1); n <= 3000; n++ { - if shouldLogSampled(n) { - logged[n] = true - } - } - for _, want := range []int64{1, 10, 100, 1000, 2000, 3000} { - require.Truef(t, logged[want], "occurrence %d should be logged", want) - } - for _, notWant := range []int64{2, 9, 11, 99, 101, 999, 1001, 1999} { - require.Falsef(t, logged[notWant], "occurrence %d should not be logged", notWant) - } - require.Len(t, logged, 6, "exactly 1, 10, 100, 1000, 2000, 3000 in the first 3000") - require.False(t, shouldLogSampled(0), "a zero count is not an occurrence") - require.False(t, shouldLogSampled(-1), "a negative count is not an occurrence") -} - -// skipWarning is one decoded skip-warning log record. -type skipWarning struct { - serviceAccountID string - code string - totalOccurrences int64 -} - -// drainForSkipWarnings runs one full List walk with its own log sink and returns -// the skip warnings that walk emitted, decoded from the JSON log records rather -// than substring-matched: total_occurrences is compared numerically, so `1` -// cannot be satisfied by `10`, `100` or `1000`, and the assertions do not rot if -// zap's encoding changes. -// -// Requires a builder whose walk visits at least one users page with data -- it -// asserts the walk did not end on its first call. -func drainForSkipWarnings(t *testing.T, builder *applicationKeyBuilder) []skipWarning { - t.Helper() - var logBuf bytes.Buffer - core := zapcore.NewCore( - zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), - zapcore.AddSync(&logBuf), - zapcore.DebugLevel, - ) - logger := zap.New(core) - ctx := ctxzap.ToContext(context.Background(), logger) - - calls := 0 - token := "" - for { - require.Less(t, calls, 200, "List did not terminate") - _, results, err := builder.List(ctx, nil, rs.SyncOpAttrs{PageToken: pagination.Token{Token: token}}) - require.NoError(t, err, "a 403 must never fail the sync") - require.NotNil(t, results) - if results.NextPageToken == "" { - // List treats an empty INCOMING token as "first call of this walk" - // and resets the sampling counter on it. That is only sound because - // a walk cannot both start and end on the same call -- otherwise an - // empty token would be ambiguous between the two. Assert it here, - // in the branch where it can actually fail; a walk that terminated - // on call zero never ran at all. - require.NotZero(t, calls, "walk ended on its first call, so an empty token is ambiguous between start and end") - break - } - token = results.NextPageToken - calls++ - } - require.NoError(t, logger.Sync()) - - var skips []skipWarning - for _, line := range strings.Split(strings.TrimSpace(logBuf.String()), "\n") { - if line == "" { - continue - } - var rec struct { - Msg string `json:"msg"` - ServiceAccountID string `json:"service_account_id"` - Code string `json:"code"` - TotalOccurrences *int64 `json:"total_occurrences"` - } - require.NoErrorf(t, json.Unmarshal([]byte(line), &rec), "log line is not JSON: %s", line) - // Matched exactly, not by substring, so a reworded production message - // fails the sampling assertions loudly instead of quietly matching - // nothing. (Inline rather than a named constant: gosec G101 reads a - // const holding this sentence as a hardcoded credential.) - if rec.Msg != "baton-datadog: skipping application keys for service account" { - continue - } - require.NotNilf(t, rec.TotalOccurrences, "skip warning must carry total_occurrences: %s", line) - skips = append(skips, skipWarning{ - serviceAccountID: rec.ServiceAccountID, - code: rec.Code, - totalOccurrences: *rec.TotalOccurrences, - }) - } - return skips -} - -// TestApplicationKeyBuilderListSamplesSkipWarning: when the configured Datadog -// role cannot read application keys org-wide, the skip warning must not fire -// once per service account (criteria L7). With 12 forbidden service accounts -// only the 1st and 10th are logged, each carrying total_occurrences. -// -// The counter also has to restart per walk. applicationKeyBuilder is built once -// per connector process (Datadog.ResourceSyncers runs inside -// connectorbuilder.NewConnector), so a counter that only ever climbed would let -// the first sync consume the 1/10/100 slots and leave every later sync silent -// until the running total reached 1000 -- which is worse than the noise the -// sampling exists to prevent. Draining twice proves the second walk gets its -// own schedule, and the occurrence numbers are compared exactly so a later -// occurrence cannot pass as the first. -func TestApplicationKeyBuilderListSamplesSkipWarning(t *testing.T) { - const serviceAccounts = 12 - entries := make([]string, 0, serviceAccounts) - forbidden := map[string]bool{} - for i := 0; i < serviceAccounts; i++ { - id := fmt.Sprintf("sa-%02d", i) - entries = append(entries, fmt.Sprintf(`{"id":"%s","type":"users","attributes":{"service_account":true}}`, id)) - forbidden[id] = true - } - usersPages := []string{"[" + strings.Join(entries, ",") + "]"} - - server, requests := newAppKeyListServer(t, usersPages, nil, forbidden) - defer server.Close() - - builder := newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)) - - for walk := 1; walk <= 2; walk++ { - skips := drainForSkipWarnings(t, builder) - require.Lenf(t, skips, 2, "walk %d: 12 skips must log only the 1st and 10th, not one line each", walk) - require.Equalf(t, int64(1), skips[0].totalOccurrences, "walk %d: first logged line must be occurrence 1, not a later one", walk) - require.Equalf(t, int64(10), skips[1].totalOccurrences, "walk %d: second logged line must be occurrence 10", walk) - for i, skip := range skips { - require.Equalf(t, codes.PermissionDenied.String(), skip.code, - "walk %d line %d: a 403 must be reported as PermissionDenied", walk, i) - require.NotEmptyf(t, skip.serviceAccountID, "walk %d line %d: skip must name the service account", walk, i) - } - } - - // Both walks attempted every service account. This is also what catches a - // walk that terminates early: a paging change that ended the walk before - // draining every service account would show up here as a short count. - attempted := 0 - for _, req := range snapshotRequests(requests) { - if strings.Contains(req.path, "/application_keys") { - attempted++ - } - } - require.Equal(t, serviceAccounts*2, attempted, "every service account must be attempted on every walk") -} - // --- application-key scopes on the resource profile ----------------------- // profileScopes reads the scopes list out of a resource profile. The second @@ -1172,8 +1075,9 @@ func TestApplicationKeyResourceMalformedCreatedAt(t *testing.T) { // TestApplicationKeyListSurvivesMalformedCreatedAt: the whole walk must complete // when the provider returns an unparseable created_at, the affected key must -// still be synced, and the warning must be sampled with total_occurrences rather -// than emitted once per key (L7). +// still be synced, and every affected key must be reported. The timestamp is +// decorative -- nothing about identifying, attributing or revoking the key +// depends on it -- so dropping the field beats losing sight of the credential. func TestApplicationKeyListSurvivesMalformedCreatedAt(t *testing.T) { const keys = 12 entries := make([]string, 0, keys) @@ -1220,17 +1124,15 @@ func TestApplicationKeyListSurvivesMalformedCreatedAt(t *testing.T) { continue } var rec struct { - Msg string `json:"msg"` - TotalOccurrences *int64 `json:"total_occurrences"` + Msg string `json:"msg"` } require.NoError(t, json.Unmarshal([]byte(line), &rec)) if rec.Msg != "baton-datadog: application key created_at could not be parsed; syncing the key without it" { continue } warned++ - require.NotNil(t, rec.TotalOccurrences, "the warning must carry total_occurrences") } - require.Equal(t, 2, warned, "12 malformed timestamps must log only the 1st and 10th, not one line each") + require.Equal(t, keys, warned, "every key with an unparseable created_at must be reported") } // TestApplicationKeySyncRequestsFullUsersPage: the users walk must ask for the @@ -1319,18 +1221,16 @@ func TestApiTokenListSurvivesMalformedTimestamps(t *testing.T) { continue } var rec struct { - Msg string `json:"msg"` - Field string `json:"field"` - TotalOccurrences *int64 `json:"total_occurrences"` + Msg string `json:"msg"` + Field string `json:"field"` } require.NoError(t, json.Unmarshal([]byte(line), &rec)) if rec.Msg != "baton-datadog: organization API key timestamp could not be parsed; syncing the key without it" { continue } warned++ - require.NotNil(t, rec.TotalOccurrences, "the warning must carry total_occurrences") require.Contains(t, []string{"created_at", "modified_at"}, rec.Field, "the warning must name the field") } - // 12 keys x 2 bad fields = 24 occurrences, so only the 1st and 10th log. - require.Equal(t, 2, warned, "24 malformed timestamps must log only the 1st and 10th, not one line each") + // Every key carries two unparseable fields, and each is reported. + require.Equal(t, keys*2, warned, "every unparseable timestamp must be reported") } diff --git a/pkg/connector/helpers.go b/pkg/connector/helpers.go index 3762df2a..a55125e5 100644 --- a/pkg/connector/helpers.go +++ b/pkg/connector/helpers.go @@ -130,24 +130,3 @@ func hasMoreAPIKeyPages(res *datadogV2.APIKeysResponse, page int64, count int64, } return count != 0 } - -// shouldLogSampled reports whether the nth occurrence (1-based) of a repeating -// event should be logged, on the logarithmic schedule the repo's review -// criteria require for warnings that can fire once per resource (L7): the 1st, -// 10th and 100th occurrence, then every 1000th. Callers pass the running total -// and put it on the record as total_occurrences, so a sampled line still says -// how many times the event really happened. -// -// Neither this repo nor the vendored baton-sdk ships a sampling helper, so this -// is the smallest thing that satisfies the criteria; it is deliberately a pure -// function of the count, with the counter owned by the caller. -func shouldLogSampled(n int64) bool { - switch { - case n <= 0: - return false - case n == 1, n == 10, n == 100: - return true - default: - return n%1000 == 0 - } -} From 4f6d21169aeb0c867749e4be21208936aa5774dd Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Wed, 26 Aug 2026 15:13:01 +0000 Subject: [PATCH 33/49] fix: distinguish 403 from 404 when service-account keys cannot be read Both codes still fail the sync. Attaching the service_account_write hint to NotFound told an operator to grant a permission they may already hold: a service account deleted between the users page and this call returns 404, and the gRPC code is a mechanical translation of the HTTP status, so NotFound carries no other meaning. Give it its own message instead. Also reword the doc comment on TestApplicationKeyBuilderListFailsHardOnUnexpectedError, which still described a skip path that no longer exists. Co-authored-by: c1-squire-dev[bot] --- pkg/connector/application_key.go | 9 ++++++++- pkg/connector/credential_lifecycle_test.go | 6 +++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pkg/connector/application_key.go b/pkg/connector/application_key.go index 38a8064b..6018aedf 100644 --- a/pkg/connector/application_key.go +++ b/pkg/connector/application_key.go @@ -209,12 +209,19 @@ func (o *applicationKeyBuilder) listApplicationKeyPage( // cannot be assumed to mean the service account is genuinely gone. resp, err := o.wrapper.ListServiceAccountApplicationKeys(ctx, serviceAccountID, page, defaultV2PageSize) if err != nil { - if code := status.Code(err); code == codes.PermissionDenied || code == codes.NotFound { + code := status.Code(err) + if code == codes.PermissionDenied { return nil, nil, fmt.Errorf( "baton-datadog: list application keys for service account %q: %w "+ "(listing service-account application keys requires the Datadog service_account_write permission)", serviceAccountID, err) } + if code == codes.NotFound { + return nil, nil, fmt.Errorf( + "baton-datadog: list application keys for service account %q: %w "+ + "(the service account was not found, and may have been deleted mid-sync)", + serviceAccountID, err) + } return nil, nil, fmt.Errorf("baton-datadog: list application keys for service account %q: %w", serviceAccountID, err) } diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index 8acde9a8..c644e5e2 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -724,9 +724,9 @@ func TestApplicationKeyBuilderListFailsHardOnMissingServiceAccount(t *testing.T) t.Fatal("List never surfaced the 404") } -// TestApplicationKeyBuilderListFailsHardOnUnexpectedError: only -// PermissionDenied/NotFound are skipped; any other provider error must still -// abort the sync rather than silently under-reporting keys. +// TestApplicationKeyBuilderListFailsHardOnUnexpectedError: every provider +// error aborts the sync; this covers the non-403/404 path, where the error +// carries no permission or missing-service-account hint of its own. func TestApplicationKeyBuilderListFailsHardOnUnexpectedError(t *testing.T) { requests := &[]recordedRequest{} server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { From 25050be3ba931c4760a718946855d107ce8ea402 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Thu, 27 Aug 2026 06:49:04 +0000 Subject: [PATCH 34/49] chore: re-trigger CI The workflow runs on 4f6d2116 never had a runner assigned and stayed queued; pr-review cannot be re-run from the Actions UI. This empty commit exists only to schedule a fresh set of runs. Co-authored-by: c1-squire-dev[bot] From a7a4f61b203307d3d8cb8db3fb8777979c727879 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:42:59 +0000 Subject: [PATCH 35/49] chore(deps): vendor the baton-sdk credential type discriminator baton-sdk#1109 splits credential shape from credential kind: the closed CredentialIssueOptionDescriptor.option enum keeps naming the shape, and the open secret_resource_type_id names which kind of that shape comes back. It also adds CredentialIssueOptions.secret_resource_type_id, which issuance requests must now set, and CredentialIssueOptionDescriptor .preferred, which names the default descriptor when several share a shape. This pins a pseudo-version of that PR's head rather than a released tag, so it must be re-pointed at a released baton-sdk tag before merge. Until then the check-versions CI job fails: .versions.yaml on main pins v0.24.4 and is managed by baton-admin, so it cannot be updated here. Co-authored-by: c1-squire-dev[bot] --- go.mod | 2 +- go.sum | 4 +- .../baton-sdk/pb/c1/c1z/v3/manifest.pb.go | 25 +- .../pb/c1/c1z/v3/manifest.pb.validate.go | 2 + .../pb/c1/c1z/v3/manifest_protoopaque.pb.go | 21 +- .../pb/c1/connector/v2/connector.pb.go | 68 +- .../c1/connector/v2/connector.pb.validate.go | 2 + .../connector/v2/connector_protoopaque.pb.go | 40 +- .../pb/c1/connector/v2/resource.pb.go | 38 +- .../c1/connector/v2/resource.pb.validate.go | 2 + .../connector/v2/resource_protoopaque.pb.go | 40 +- .../pb/c1/connectorapi/baton/v1/baton.pb.go | 272 ++- .../baton/v1/baton.pb.validate.go | 41 - .../baton/v1/baton_protoopaque.pb.go | 260 +-- .../baton-sdk/pb/c1/storage/v3/records.pb.go | 138 +- .../pb/c1/storage/v3/records.pb.validate.go | 8 +- .../c1/storage/v3/records_protoopaque.pb.go | 150 +- .../baton-sdk/pkg/cli/commands.go | 8 - .../baton-sdk/pkg/config/config.go | 7 +- .../pkg/connectorbuilder/connectorbuilder.go | 57 +- .../credential_issue_validation.go | 67 +- .../baton-sdk/pkg/connectorrunner/runner.go | 20 - .../pkg/connectorstore/connectorstore.go | 40 +- .../baton-sdk/pkg/crypto/providers/age/age.go | 21 +- .../baton-sdk/pkg/dotc1z/c1file.go | 10 +- .../baton-sdk/pkg/dotc1z/c1file_attached.go | 274 --- .../baton-sdk/pkg/dotc1z/c1file_store.go | 7 +- .../baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go | 16 +- .../pkg/dotc1z/c1zstore/cleanup_policy.go | 37 +- .../baton-sdk/pkg/dotc1z/c1zstore/engine.go | 11 +- .../baton-sdk/pkg/dotc1z/c1zstore/file_ops.go | 5 - .../pkg/dotc1z/c1zstore/sync_meta.go | 29 +- .../conductorone/baton-sdk/pkg/dotc1z/diff.go | 124 -- .../pkg/dotc1z/engine/pebble/adapter.go | 26 +- .../pkg/dotc1z/engine/pebble/adapter_diff.go | 21 - .../dotc1z/engine/pebble/adapter_file_ops.go | 11 +- .../engine/pebble/adapter_grants_store.go | 6 +- .../dotc1z/engine/pebble/adapter_reader.go | 43 +- .../dotc1z/engine/pebble/adapter_sync_meta.go | 21 +- .../pkg/dotc1z/engine/pebble/bulk_import.go | 12 + .../pkg/dotc1z/engine/pebble/cleanup.go | 12 + .../pkg/dotc1z/engine/pebble/digest.go | 13 + .../pkg/dotc1z/engine/pebble/engine.go | 109 +- .../pebble/entitlement_graph_sidecar.go | 88 + .../pkg/dotc1z/engine/pebble/entitlements.go | 128 +- .../pkg/dotc1z/engine/pebble/grant_digest.go | 84 +- .../pkg/dotc1z/engine/pebble/grants.go | 137 +- .../pkg/dotc1z/engine/pebble/if_newer.go | 293 --- .../dotc1z/engine/pebble/index_migrations.go | 8 + .../engine/pebble/internal/rawdb/families.go | 124 +- .../engine/pebble/internal/rawdb/keyspace.go | 160 +- .../engine/pebble/internal/rawdb/rawdb.go | 141 +- .../engine/pebble/internal/rawdb/records.go | 249 ++- .../pkg/dotc1z/engine/pebble/keys.go | 73 + .../pkg/dotc1z/engine/pebble/lookup.go | 21 +- .../pkg/dotc1z/engine/pebble/manifest.go | 1 + .../pkg/dotc1z/engine/pebble/raw_records.go | 102 - .../pkg/dotc1z/engine/pebble/source_cache.go | 1783 +++++++++++++++++ .../pkg/dotc1z/engine/pebble/sync_runs.go | 4 +- .../pkg/dotc1z/engine/pebble/test_seams.go | 36 + .../pkg/dotc1z/engine/pebble/translate_v2.go | 36 +- .../baton-sdk/pkg/dotc1z/engine_registry.go | 22 +- .../pkg/dotc1z/format/v3/manifest.go | 24 +- .../baton-sdk/pkg/dotc1z/pebble_store.go | 113 +- .../baton-sdk/pkg/dotc1z/source_cache.go | 412 ++++ .../baton-sdk/pkg/dotc1z/sql_helpers.go | 6 +- .../baton-sdk/pkg/dotc1z/sync_runs.go | 75 +- .../baton-sdk/pkg/dotc1z/to_pebble.go | 27 +- .../conductorone/baton-sdk/pkg/exit/exit.go | 53 + .../pkg/field/default_relationships.go | 2 - .../baton-sdk/pkg/field/defaults.go | 26 +- .../baton-sdk/pkg/provisioner/provisioner.go | 52 +- .../conductorone/baton-sdk/pkg/sdk/version.go | 2 +- .../baton-sdk/pkg/sourcecache/context.go | 17 + .../baton-sdk/pkg/sourcecache/sourcecache.go | 148 ++ .../baton-sdk/pkg/sync/expand/graph.go | 107 + .../baton-sdk/pkg/sync/expand/graph_blob.go | 91 + .../baton-sdk/pkg/sync/expand/incremental.go | 590 ++++++ .../pkg/sync/expand/topological_merge.go | 6 +- .../pkg/sync/external_principal_index.go | 2 +- .../baton-sdk/pkg/sync/ingest_invariants.go | 55 +- .../baton-sdk/pkg/sync/parallel_syncer.go | 12 +- .../conductorone/baton-sdk/pkg/sync/state.go | 89 + .../conductorone/baton-sdk/pkg/sync/syncer.go | 177 +- .../pkg/synccompactor/attached/attached.go | 13 +- .../baton-sdk/pkg/synccompactor/compactor.go | 843 +++++++- .../pkg/synccompactor/compactor_pebble.go | 153 +- .../pkg/synccompactor/pebble/bucket_plans.go | 16 +- .../baton-sdk/pkg/synccompactor/pebble/doc.go | 6 +- .../pkg/synccompactor/pebble/fold_commit.go | 21 + .../pkg/synccompactor/pebble/kway.go | 50 +- .../pkg/synccompactor/pebble/merge.go | 83 +- .../pkg/synccompactor/pebble/overlay.go | 50 +- .../baton-sdk/pkg/tasks/local/differ.go | 84 - .../conductorone/baton-sdk/pkg/tasks/tasks.go | 4 - .../baton-sdk/pkg/types/tasks/tasks.go | 4 +- .../baton-sdk/pkg/uhttp/client.go | 64 +- .../baton-sdk/pkg/uhttp/dbcache.go | 8 +- .../baton-sdk/pkg/uhttp/gocache.go | 12 +- .../baton-sdk/pkg/uhttp/wrapper.go | 30 +- vendor/modules.txt | 4 +- 101 files changed, 7073 insertions(+), 2068 deletions(-) delete mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/diff.go delete mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_diff.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlement_graph_sidecar.go delete mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/source_cache.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/exit/exit.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/context.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph_blob.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/incremental.go create mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/fold_commit.go delete mode 100644 vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/differ.go diff --git a/go.mod b/go.mod index 5cd49671..5db1fa0d 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.25.2 require ( github.com/DataDog/datadog-api-client-go/v2 v2.43.0 - github.com/conductorone/baton-sdk v0.24.4 + github.com/conductorone/baton-sdk v0.25.2-0.20260827221151-1ff7ce2d6fda github.com/ennyjfrick/ruleguard-logfatal v0.0.2 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/quasilyte/go-ruleguard/dsl v0.3.23 diff --git a/go.sum b/go.sum index a04280aa..458a7ea2 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b h1:VXvSNzmr8hMj8 github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/conductorone/baton-sdk v0.24.4 h1:W2YogjlYQDp1Mwt/RBiPKLWalXfANEpzg8FLc4c3bvI= -github.com/conductorone/baton-sdk v0.24.4/go.mod h1:SKm95z4KkQ23Tufo2ys88lVzbwKb0AQEbKee5GE0Lig= +github.com/conductorone/baton-sdk v0.25.2-0.20260827221151-1ff7ce2d6fda h1:vap+5POBHWfabs1h60lP3rXv9Mt209oE7kw857FxyZM= +github.com/conductorone/baton-sdk v0.25.2-0.20260827221151-1ff7ce2d6fda/go.mod h1:SKm95z4KkQ23Tufo2ys88lVzbwKb0AQEbKee5GE0Lig= github.com/conductorone/dpop v0.2.6 h1:fakwai/Xm2b/fcDUwJN41WtcSI/2UhQOyRIVvnnrrNA= github.com/conductorone/dpop v0.2.6/go.mod h1:gyo8TtzB9SCFCsjsICH4IaLZ7y64CcrDXMOPBwfq/3s= github.com/conductorone/dpop/integrations/dpop_grpc v0.2.4 h1:lYxYi9/WTSL9sE96CO0QF2BY3kehs8dTTApI134TGCA= diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.go index dcda1a7e..76056317 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.go @@ -977,7 +977,10 @@ type SyncRunSummary struct { // planning, tooling) get row counts from the envelope header without // unpacking the payload. Absent for syncs whose sidecar was never // written (e.g. interrupted syncs). - Stats *v3.SyncStatsRecord `protobuf:"bytes,6,opt,name=stats,proto3" json:"stats,omitempty"` + Stats *v3.SyncStatsRecord `protobuf:"bytes,6,opt,name=stats,proto3" json:"stats,omitempty"` + // Projection of SyncRunRecord.compacted. Header-only readers use this + // eligibility bit without unpacking or decompressing the Pebble payload. + Compacted bool `protobuf:"varint,7,opt,name=compacted,proto3" json:"compacted,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1049,6 +1052,13 @@ func (x *SyncRunSummary) GetStats() *v3.SyncStatsRecord { return nil } +func (x *SyncRunSummary) GetCompacted() bool { + if x != nil { + return x.Compacted + } + return false +} + func (x *SyncRunSummary) SetSyncId(v string) { x.SyncId = v } @@ -1073,6 +1083,10 @@ func (x *SyncRunSummary) SetStats(v *v3.SyncStatsRecord) { x.Stats = v } +func (x *SyncRunSummary) SetCompacted(v bool) { + x.Compacted = v +} + func (x *SyncRunSummary) HasStartedAt() bool { if x == nil { return false @@ -1120,6 +1134,9 @@ type SyncRunSummary_builder struct { // unpacking the payload. Absent for syncs whose sidecar was never // written (e.g. interrupted syncs). Stats *v3.SyncStatsRecord + // Projection of SyncRunRecord.compacted. Header-only readers use this + // eligibility bit without unpacking or decompressing the Pebble payload. + Compacted bool } func (b0 SyncRunSummary_builder) Build() *SyncRunSummary { @@ -1132,6 +1149,7 @@ func (b0 SyncRunSummary_builder) Build() *SyncRunSummary { x.EndedAt = b.EndedAt x.ParentSyncId = b.ParentSyncId x.Stats = b.Stats + x.Compacted = b.Compacted return m0 } @@ -1253,7 +1271,7 @@ const file_c1_c1z_v3_manifest_proto_rawDesc = "" + "\x0eRecordTypeInfo\x12*\n" + "\x11message_full_name\x18\x01 \x01(\tR\x0fmessageFullName\x12%\n" + "\x0eschema_version\x18\x02 \x01(\rR\rschemaVersion\x12'\n" + - "\x0festimated_count\x18\x03 \x01(\x03R\x0eestimatedCount\"\xa4\x02\n" + + "\x0festimated_count\x18\x03 \x01(\x03R\x0eestimatedCount\"\xc2\x02\n" + "\x0eSyncRunSummary\x12\x17\n" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12+\n" + "\x04type\x18\x02 \x01(\x0e2\x17.c1.storage.v3.SyncTypeR\x04type\x129\n" + @@ -1261,7 +1279,8 @@ const file_c1_c1z_v3_manifest_proto_rawDesc = "" + "started_at\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\tstartedAt\x125\n" + "\bended_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\aendedAt\x12$\n" + "\x0eparent_sync_id\x18\x05 \x01(\tR\fparentSyncId\x124\n" + - "\x05stats\x18\x06 \x01(\v2\x1e.c1.storage.v3.SyncStatsRecordR\x05stats\"p\n" + + "\x05stats\x18\x06 \x01(\v2\x1e.c1.storage.v3.SyncStatsRecordR\x05stats\x12\x1c\n" + + "\tcompacted\x18\a \x01(\bR\tcompacted\"p\n" + "\x12PebbleEngineConfig\x120\n" + "\x14format_major_version\x18\x01 \x01(\rR\x12formatMajorVersion\x12(\n" + "\x10cache_size_bytes\x18\x02 \x01(\x04R\x0ecacheSizeBytes*\x96\x01\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.validate.go index 432a659f..ce60b70f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.validate.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest.pb.validate.go @@ -887,6 +887,8 @@ func (m *SyncRunSummary) validate(all bool) error { } } + // no validation rules for Compacted + if len(errors) > 0 { return SyncRunSummaryMultiError(errors) } diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest_protoopaque.pb.go index 1732c6d2..0823871f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/c1z/v3/manifest_protoopaque.pb.go @@ -892,6 +892,7 @@ type SyncRunSummary struct { xxx_hidden_EndedAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=ended_at,json=endedAt,proto3"` xxx_hidden_ParentSyncId string `protobuf:"bytes,5,opt,name=parent_sync_id,json=parentSyncId,proto3"` xxx_hidden_Stats *v3.SyncStatsRecord `protobuf:"bytes,6,opt,name=stats,proto3"` + xxx_hidden_Compacted bool `protobuf:"varint,7,opt,name=compacted,proto3"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -963,6 +964,13 @@ func (x *SyncRunSummary) GetStats() *v3.SyncStatsRecord { return nil } +func (x *SyncRunSummary) GetCompacted() bool { + if x != nil { + return x.xxx_hidden_Compacted + } + return false +} + func (x *SyncRunSummary) SetSyncId(v string) { x.xxx_hidden_SyncId = v } @@ -987,6 +995,10 @@ func (x *SyncRunSummary) SetStats(v *v3.SyncStatsRecord) { x.xxx_hidden_Stats = v } +func (x *SyncRunSummary) SetCompacted(v bool) { + x.xxx_hidden_Compacted = v +} + func (x *SyncRunSummary) HasStartedAt() bool { if x == nil { return false @@ -1034,6 +1046,9 @@ type SyncRunSummary_builder struct { // unpacking the payload. Absent for syncs whose sidecar was never // written (e.g. interrupted syncs). Stats *v3.SyncStatsRecord + // Projection of SyncRunRecord.compacted. Header-only readers use this + // eligibility bit without unpacking or decompressing the Pebble payload. + Compacted bool } func (b0 SyncRunSummary_builder) Build() *SyncRunSummary { @@ -1046,6 +1061,7 @@ func (b0 SyncRunSummary_builder) Build() *SyncRunSummary { x.xxx_hidden_EndedAt = b.EndedAt x.xxx_hidden_ParentSyncId = b.ParentSyncId x.xxx_hidden_Stats = b.Stats + x.xxx_hidden_Compacted = b.Compacted return m0 } @@ -1164,7 +1180,7 @@ const file_c1_c1z_v3_manifest_proto_rawDesc = "" + "\x0eRecordTypeInfo\x12*\n" + "\x11message_full_name\x18\x01 \x01(\tR\x0fmessageFullName\x12%\n" + "\x0eschema_version\x18\x02 \x01(\rR\rschemaVersion\x12'\n" + - "\x0festimated_count\x18\x03 \x01(\x03R\x0eestimatedCount\"\xa4\x02\n" + + "\x0festimated_count\x18\x03 \x01(\x03R\x0eestimatedCount\"\xc2\x02\n" + "\x0eSyncRunSummary\x12\x17\n" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12+\n" + "\x04type\x18\x02 \x01(\x0e2\x17.c1.storage.v3.SyncTypeR\x04type\x129\n" + @@ -1172,7 +1188,8 @@ const file_c1_c1z_v3_manifest_proto_rawDesc = "" + "started_at\x18\x03 \x01(\v2\x1a.google.protobuf.TimestampR\tstartedAt\x125\n" + "\bended_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\aendedAt\x12$\n" + "\x0eparent_sync_id\x18\x05 \x01(\tR\fparentSyncId\x124\n" + - "\x05stats\x18\x06 \x01(\v2\x1e.c1.storage.v3.SyncStatsRecordR\x05stats\"p\n" + + "\x05stats\x18\x06 \x01(\v2\x1e.c1.storage.v3.SyncStatsRecordR\x05stats\x12\x1c\n" + + "\tcompacted\x18\a \x01(\bR\tcompacted\"p\n" + "\x12PebbleEngineConfig\x120\n" + "\x14format_major_version\x18\x01 \x01(\rR\x12formatMajorVersion\x12(\n" + "\x10cache_size_bytes\x18\x02 \x01(\x04R\x0ecacheSizeBytes*\x96\x01\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/connector.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/connector.pb.go index 968a34a6..ab37e7d4 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/connector.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/connector.pb.go @@ -760,9 +760,18 @@ func (b0 CredentialDetailsCredentialRotation_builder) Build() *CredentialDetails // Advertises which credential options CredentialManagerService.IssueCredential // supports for this identity type. type CredentialDetailsCredentialIssue struct { - state protoimpl.MessageState `protogen:"hybrid.v1"` - Options []*CredentialIssueOptionDescriptor `protobuf:"bytes,1,rep,name=options,proto3" json:"options,omitempty"` - PreferredOption CapabilityDetailCredentialOption `protobuf:"varint,2,opt,name=preferred_option,json=preferredOption,proto3,enum=c1.connector.v2.CapabilityDetailCredentialOption" json:"preferred_option,omitempty"` + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Advertised issuance options, unique on (option, secret_resource_type_id). + // Several descriptors may share an option when they mint different secret + // resource types -- an organization-wide API key and a per-identity API key, + // for example. + Options []*CredentialIssueOptionDescriptor `protobuf:"bytes,1,rep,name=options,proto3" json:"options,omitempty"` + // The preferred credential shape. Selection is two-level: this field picks + // the shape and CredentialIssueOptionDescriptor.preferred picks the + // descriptor within it. To resolve the default, look only at the descriptors + // carrying this option: one of them has preferred set, except where the + // option has a single descriptor, which is the default with the flag unset. + PreferredOption CapabilityDetailCredentialOption `protobuf:"varint,2,opt,name=preferred_option,json=preferredOption,proto3,enum=c1.connector.v2.CapabilityDetailCredentialOption" json:"preferred_option,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -817,7 +826,16 @@ func (x *CredentialDetailsCredentialIssue) SetPreferredOption(v CapabilityDetail type CredentialDetailsCredentialIssue_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - Options []*CredentialIssueOptionDescriptor + // Advertised issuance options, unique on (option, secret_resource_type_id). + // Several descriptors may share an option when they mint different secret + // resource types -- an organization-wide API key and a per-identity API key, + // for example. + Options []*CredentialIssueOptionDescriptor + // The preferred credential shape. Selection is two-level: this field picks + // the shape and CredentialIssueOptionDescriptor.preferred picks the + // descriptor within it. To resolve the default, look only at the descriptors + // carrying this option: one of them has preferred set, except where the + // option has a single descriptor, which is the default with the flag unset. PreferredOption CapabilityDetailCredentialOption } @@ -842,10 +860,18 @@ type CredentialIssueOptionDescriptor struct { ResourceMode CredentialResourceMode `protobuf:"varint,8,opt,name=resource_mode,json=resourceMode,proto3,enum=c1.connector.v2.CredentialResourceMode" json:"resource_mode,omitempty"` // Resource type returned by IssueCredential. It must be registered with a // ResourceDeleterV2 so every issued credential has a provider revoke path, - // including virtual credentials that cannot be listed later. + // including virtual credentials that cannot be listed later. Together with + // option it identifies this descriptor, so two credential kinds sharing one + // shape must return distinct resource types. SecretResourceTypeId string `protobuf:"bytes,9,opt,name=secret_resource_type_id,json=secretResourceTypeId,proto3" json:"secret_resource_type_id,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Marks this descriptor the default within its option, for a caller + // presenting a choice. At most one descriptor per option may set it, and one + // must whenever several descriptors share that option: a default taken from + // declaration order would not be stable. It selects nothing at issue time -- + // CredentialIssueOptions.secret_resource_type_id is still required. + Preferred bool `protobuf:"varint,10,opt,name=preferred,proto3" json:"preferred,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CredentialIssueOptionDescriptor) Reset() { @@ -936,6 +962,13 @@ func (x *CredentialIssueOptionDescriptor) GetSecretResourceTypeId() string { return "" } +func (x *CredentialIssueOptionDescriptor) GetPreferred() bool { + if x != nil { + return x.Preferred + } + return false +} + func (x *CredentialIssueOptionDescriptor) SetOption(v CapabilityDetailCredentialOption) { x.Option = v } @@ -972,6 +1005,10 @@ func (x *CredentialIssueOptionDescriptor) SetSecretResourceTypeId(v string) { x.SecretResourceTypeId = v } +func (x *CredentialIssueOptionDescriptor) SetPreferred(v bool) { + x.Preferred = v +} + func (x *CredentialIssueOptionDescriptor) HasExpiry() bool { if x == nil { return false @@ -996,8 +1033,16 @@ type CredentialIssueOptionDescriptor_builder struct { ResourceMode CredentialResourceMode // Resource type returned by IssueCredential. It must be registered with a // ResourceDeleterV2 so every issued credential has a provider revoke path, - // including virtual credentials that cannot be listed later. + // including virtual credentials that cannot be listed later. Together with + // option it identifies this descriptor, so two credential kinds sharing one + // shape must return distinct resource types. SecretResourceTypeId string + // Marks this descriptor the default within its option, for a caller + // presenting a choice. At most one descriptor per option may set it, and one + // must whenever several descriptors share that option: a default taken from + // declaration order would not be stable. It selects nothing at issue time -- + // CredentialIssueOptions.secret_resource_type_id is still required. + Preferred bool } func (b0 CredentialIssueOptionDescriptor_builder) Build() *CredentialIssueOptionDescriptor { @@ -1013,6 +1058,7 @@ func (b0 CredentialIssueOptionDescriptor_builder) Build() *CredentialIssueOption x.Audiences = b.Audiences x.ResourceMode = b.ResourceMode x.SecretResourceTypeId = b.SecretResourceTypeId + x.Preferred = b.Preferred return m0 } @@ -2670,7 +2716,7 @@ const file_c1_connector_v2_connector_proto_rawDesc = "" + "\x1bpreferred_credential_option\x18\x02 \x01(\x0e21.c1.connector.v2.CapabilityDetailCredentialOptionR\x19preferredCredentialOption\"\xcc\x01\n" + " CredentialDetailsCredentialIssue\x12J\n" + "\aoptions\x18\x01 \x03(\v20.c1.connector.v2.CredentialIssueOptionDescriptorR\aoptions\x12\\\n" + - "\x10preferred_option\x18\x02 \x01(\x0e21.c1.connector.v2.CapabilityDetailCredentialOptionR\x0fpreferredOption\"\xae\x04\n" + + "\x10preferred_option\x18\x02 \x01(\x0e21.c1.connector.v2.CapabilityDetailCredentialOptionR\x0fpreferredOption\"\xcc\x04\n" + "\x1fCredentialIssueOptionDescriptor\x12I\n" + "\x06option\x18\x01 \x01(\x0e21.c1.connector.v2.CapabilityDetailCredentialOptionR\x06option\x12H\n" + "\fkey_profiles\x18\x02 \x03(\v2%.c1.connector.v2.KeyGenerationProfileR\vkeyProfiles\x12A\n" + @@ -2681,7 +2727,9 @@ const file_c1_connector_v2_connector_proto_rawDesc = "" + "\taudiences\x18\a \x03(\tR\taudiences\x12L\n" + "\rresource_mode\x18\b \x01(\x0e2'.c1.connector.v2.CredentialResourceModeR\fresourceMode\x12A\n" + "\x17secret_resource_type_id\x18\t \x01(\tB\n" + - "\xfaB\ar\x05 \x01(\x80\bR\x14secretResourceTypeId\"t\n" + + "\xfaB\ar\x05 \x01(\x80\bR\x14secretResourceTypeId\x12\x1c\n" + + "\tpreferred\x18\n" + + " \x01(\bR\tpreferred\"t\n" + "\x18IssuanceExpiryCapability\x12+\n" + "\x03min\x18\x01 \x01(\v2\x19.google.protobuf.DurationR\x03min\x12+\n" + "\x03max\x18\x02 \x01(\v2\x19.google.protobuf.DurationR\x03max\"\xa5\x02\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/connector.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/connector.pb.validate.go index a8ae70e2..cbf7121e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/connector.pb.validate.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/connector.pb.validate.go @@ -1297,6 +1297,8 @@ func (m *CredentialIssueOptionDescriptor) validate(all bool) error { errors = append(errors, err) } + // no validation rules for Preferred + if len(errors) > 0 { return CredentialIssueOptionDescriptorMultiError(errors) } diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/connector_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/connector_protoopaque.pb.go index 08d42ba7..3cf62a35 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/connector_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/connector_protoopaque.pb.go @@ -825,7 +825,16 @@ func (x *CredentialDetailsCredentialIssue) SetPreferredOption(v CapabilityDetail type CredentialDetailsCredentialIssue_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - Options []*CredentialIssueOptionDescriptor + // Advertised issuance options, unique on (option, secret_resource_type_id). + // Several descriptors may share an option when they mint different secret + // resource types -- an organization-wide API key and a per-identity API key, + // for example. + Options []*CredentialIssueOptionDescriptor + // The preferred credential shape. Selection is two-level: this field picks + // the shape and CredentialIssueOptionDescriptor.preferred picks the + // descriptor within it. To resolve the default, look only at the descriptors + // carrying this option: one of them has preferred set, except where the + // option has a single descriptor, which is the default with the flag unset. PreferredOption CapabilityDetailCredentialOption } @@ -849,6 +858,7 @@ type CredentialIssueOptionDescriptor struct { xxx_hidden_Audiences []string `protobuf:"bytes,7,rep,name=audiences,proto3"` xxx_hidden_ResourceMode CredentialResourceMode `protobuf:"varint,8,opt,name=resource_mode,json=resourceMode,proto3,enum=c1.connector.v2.CredentialResourceMode"` xxx_hidden_SecretResourceTypeId string `protobuf:"bytes,9,opt,name=secret_resource_type_id,json=secretResourceTypeId,proto3"` + xxx_hidden_Preferred bool `protobuf:"varint,10,opt,name=preferred,proto3"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -943,6 +953,13 @@ func (x *CredentialIssueOptionDescriptor) GetSecretResourceTypeId() string { return "" } +func (x *CredentialIssueOptionDescriptor) GetPreferred() bool { + if x != nil { + return x.xxx_hidden_Preferred + } + return false +} + func (x *CredentialIssueOptionDescriptor) SetOption(v CapabilityDetailCredentialOption) { x.xxx_hidden_Option = v } @@ -979,6 +996,10 @@ func (x *CredentialIssueOptionDescriptor) SetSecretResourceTypeId(v string) { x.xxx_hidden_SecretResourceTypeId = v } +func (x *CredentialIssueOptionDescriptor) SetPreferred(v bool) { + x.xxx_hidden_Preferred = v +} + func (x *CredentialIssueOptionDescriptor) HasExpiry() bool { if x == nil { return false @@ -1003,8 +1024,16 @@ type CredentialIssueOptionDescriptor_builder struct { ResourceMode CredentialResourceMode // Resource type returned by IssueCredential. It must be registered with a // ResourceDeleterV2 so every issued credential has a provider revoke path, - // including virtual credentials that cannot be listed later. + // including virtual credentials that cannot be listed later. Together with + // option it identifies this descriptor, so two credential kinds sharing one + // shape must return distinct resource types. SecretResourceTypeId string + // Marks this descriptor the default within its option, for a caller + // presenting a choice. At most one descriptor per option may set it, and one + // must whenever several descriptors share that option: a default taken from + // declaration order would not be stable. It selects nothing at issue time -- + // CredentialIssueOptions.secret_resource_type_id is still required. + Preferred bool } func (b0 CredentialIssueOptionDescriptor_builder) Build() *CredentialIssueOptionDescriptor { @@ -1020,6 +1049,7 @@ func (b0 CredentialIssueOptionDescriptor_builder) Build() *CredentialIssueOption x.xxx_hidden_Audiences = b.Audiences x.xxx_hidden_ResourceMode = b.ResourceMode x.xxx_hidden_SecretResourceTypeId = b.SecretResourceTypeId + x.xxx_hidden_Preferred = b.Preferred return m0 } @@ -2693,7 +2723,7 @@ const file_c1_connector_v2_connector_proto_rawDesc = "" + "\x1bpreferred_credential_option\x18\x02 \x01(\x0e21.c1.connector.v2.CapabilityDetailCredentialOptionR\x19preferredCredentialOption\"\xcc\x01\n" + " CredentialDetailsCredentialIssue\x12J\n" + "\aoptions\x18\x01 \x03(\v20.c1.connector.v2.CredentialIssueOptionDescriptorR\aoptions\x12\\\n" + - "\x10preferred_option\x18\x02 \x01(\x0e21.c1.connector.v2.CapabilityDetailCredentialOptionR\x0fpreferredOption\"\xae\x04\n" + + "\x10preferred_option\x18\x02 \x01(\x0e21.c1.connector.v2.CapabilityDetailCredentialOptionR\x0fpreferredOption\"\xcc\x04\n" + "\x1fCredentialIssueOptionDescriptor\x12I\n" + "\x06option\x18\x01 \x01(\x0e21.c1.connector.v2.CapabilityDetailCredentialOptionR\x06option\x12H\n" + "\fkey_profiles\x18\x02 \x03(\v2%.c1.connector.v2.KeyGenerationProfileR\vkeyProfiles\x12A\n" + @@ -2704,7 +2734,9 @@ const file_c1_connector_v2_connector_proto_rawDesc = "" + "\taudiences\x18\a \x03(\tR\taudiences\x12L\n" + "\rresource_mode\x18\b \x01(\x0e2'.c1.connector.v2.CredentialResourceModeR\fresourceMode\x12A\n" + "\x17secret_resource_type_id\x18\t \x01(\tB\n" + - "\xfaB\ar\x05 \x01(\x80\bR\x14secretResourceTypeId\"t\n" + + "\xfaB\ar\x05 \x01(\x80\bR\x14secretResourceTypeId\x12\x1c\n" + + "\tpreferred\x18\n" + + " \x01(\bR\tpreferred\"t\n" + "\x18IssuanceExpiryCapability\x12+\n" + "\x03min\x18\x01 \x01(\v2\x19.google.protobuf.DurationR\x03min\x12+\n" + "\x03max\x18\x02 \x01(\v2\x19.google.protobuf.DurationR\x03max\"\xa5\x02\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource.pb.go index 487b0dc5..5cec01e5 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource.pb.go @@ -1831,6 +1831,13 @@ func (*KeyGenerationProfile_Crv) isKeyGenerationProfile_Parameters() {} // adding issuance-only arms must not expand account creation or rotation. type CredentialIssueOptions struct { state protoimpl.MessageState `protogen:"hybrid.v1"` + // Which of the connector's advertised issuance options to select, naming the + // resource type the minted credential comes back AS and matching + // CredentialIssueOptionDescriptor.secret_resource_type_id. Required: together + // with the options arm below it identifies exactly one descriptor. Carries no + // validate.rules because nothing on the issuance path calls the generated + // Validate(); the SDK enforces presence and length in Go instead. + SecretResourceTypeId string `protobuf:"bytes,1,opt,name=secret_resource_type_id,json=secretResourceTypeId,proto3" json:"secret_resource_type_id,omitempty"` // Types that are valid to be assigned to Options: // // *CredentialIssueOptions_ApiKey_ @@ -1867,6 +1874,13 @@ func (x *CredentialIssueOptions) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } +func (x *CredentialIssueOptions) GetSecretResourceTypeId() string { + if x != nil { + return x.SecretResourceTypeId + } + return "" +} + func (x *CredentialIssueOptions) GetOptions() isCredentialIssueOptions_Options { if x != nil { return x.Options @@ -1910,6 +1924,10 @@ func (x *CredentialIssueOptions) GetClientSecret() *CredentialIssueOptions_Clien return nil } +func (x *CredentialIssueOptions) SetSecretResourceTypeId(v string) { + x.SecretResourceTypeId = v +} + func (x *CredentialIssueOptions) SetApiKey(v *CredentialIssueOptions_ApiKey) { if v == nil { x.Options = nil @@ -2036,6 +2054,13 @@ func (x *CredentialIssueOptions) WhichOptions() case_CredentialIssueOptions_Opti type CredentialIssueOptions_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + // Which of the connector's advertised issuance options to select, naming the + // resource type the minted credential comes back AS and matching + // CredentialIssueOptionDescriptor.secret_resource_type_id. Required: together + // with the options arm below it identifies exactly one descriptor. Carries no + // validate.rules because nothing on the issuance path calls the generated + // Validate(); the SDK enforces presence and length in Go instead. + SecretResourceTypeId string // Fields of oneof Options: ApiKey *CredentialIssueOptions_ApiKey Keypair *CredentialIssueOptions_Keypair @@ -2048,6 +2073,7 @@ func (b0 CredentialIssueOptions_builder) Build() *CredentialIssueOptions { m0 := &CredentialIssueOptions{} b, x := &b0, m0 _, _ = b, x + x.SecretResourceTypeId = b.SecretResourceTypeId if b.ApiKey != nil { x.Options = &CredentialIssueOptions_ApiKey_{b.ApiKey} } @@ -5889,8 +5915,11 @@ func (b0 EncryptionConfig_JWKPublicKeyConfig_builder) Build() *EncryptionConfig_ // supported by the configured age provider. EncryptedData.encrypted_bytes // contains a standard binary age file when this config is used. The provider // sets EncryptedData.key_ids to one lowercase hexadecimal SHA-256 digest of -// the UTF-8 canonical recipient string. It leaves the deprecated -// EncryptedData.key_id empty. +// the UTF-8 canonical recipient string, and leaves the deprecated +// EncryptedData.key_id empty. Rather than reimplement this derivation, +// consumers written in Go should call +// pkg/crypto/providers/age.KeyIDForRecipient, which is the single source of +// truth for the convention. type EncryptionConfig_AgeRecipientConfig struct { state protoimpl.MessageState `protogen:"hybrid.v1"` Recipient string `protobuf:"bytes,1,opt,name=recipient,proto3" json:"recipient,omitempty"` @@ -6050,8 +6079,9 @@ const file_c1_connector_v2_resource_proto_rawDesc = "" + "\x10rsa_modulus_bits\x18\x02 \x01(\rH\x00R\x0ersaModulusBits\x12\x1b\n" + "\x03crv\x18\x03 \x01(\tB\a\xfaB\x04r\x02(@H\x00R\x03crvB\f\n" + "\n" + - "parameters\"\x9c\x04\n" + - "\x16CredentialIssueOptions\x12I\n" + + "parameters\"\xd3\x04\n" + + "\x16CredentialIssueOptions\x125\n" + + "\x17secret_resource_type_id\x18\x01 \x01(\tR\x14secretResourceTypeId\x12I\n" + "\aapi_key\x18d \x01(\v2..c1.connector.v2.CredentialIssueOptions.ApiKeyH\x00R\x06apiKey\x12K\n" + "\akeypair\x18e \x01(\v2/.c1.connector.v2.CredentialIssueOptions.KeypairH\x00R\akeypair\x12E\n" + "\x05token\x18f \x01(\v2-.c1.connector.v2.CredentialIssueOptions.TokenH\x00R\x05token\x12[\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource.pb.validate.go index d91b716a..703357c5 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource.pb.validate.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource.pb.validate.go @@ -2719,6 +2719,8 @@ func (m *CredentialIssueOptions) validate(all bool) error { var errors []error + // no validation rules for SecretResourceTypeId + switch v := m.Options.(type) { case *CredentialIssueOptions_ApiKey_: if v == nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource_protoopaque.pb.go index d3c999ee..2e2bf9b7 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource_protoopaque.pb.go @@ -1831,10 +1831,11 @@ func (*keyGenerationProfile_Crv) isKeyGenerationProfile_Parameters() {} // CredentialIssueOptions are intentionally separate from CredentialOptions: // adding issuance-only arms must not expand account creation or rotation. type CredentialIssueOptions struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_Options isCredentialIssueOptions_Options `protobuf_oneof:"options"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_SecretResourceTypeId string `protobuf:"bytes,1,opt,name=secret_resource_type_id,json=secretResourceTypeId,proto3"` + xxx_hidden_Options isCredentialIssueOptions_Options `protobuf_oneof:"options"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *CredentialIssueOptions) Reset() { @@ -1862,6 +1863,13 @@ func (x *CredentialIssueOptions) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } +func (x *CredentialIssueOptions) GetSecretResourceTypeId() string { + if x != nil { + return x.xxx_hidden_SecretResourceTypeId + } + return "" +} + func (x *CredentialIssueOptions) GetApiKey() *CredentialIssueOptions_ApiKey { if x != nil { if x, ok := x.xxx_hidden_Options.(*credentialIssueOptions_ApiKey_); ok { @@ -1898,6 +1906,10 @@ func (x *CredentialIssueOptions) GetClientSecret() *CredentialIssueOptions_Clien return nil } +func (x *CredentialIssueOptions) SetSecretResourceTypeId(v string) { + x.xxx_hidden_SecretResourceTypeId = v +} + func (x *CredentialIssueOptions) SetApiKey(v *CredentialIssueOptions_ApiKey) { if v == nil { x.xxx_hidden_Options = nil @@ -2024,6 +2036,13 @@ func (x *CredentialIssueOptions) WhichOptions() case_CredentialIssueOptions_Opti type CredentialIssueOptions_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + // Which of the connector's advertised issuance options to select, naming the + // resource type the minted credential comes back AS and matching + // CredentialIssueOptionDescriptor.secret_resource_type_id. Required: together + // with the options arm below it identifies exactly one descriptor. Carries no + // validate.rules because nothing on the issuance path calls the generated + // Validate(); the SDK enforces presence and length in Go instead. + SecretResourceTypeId string // Fields of oneof xxx_hidden_Options: ApiKey *CredentialIssueOptions_ApiKey Keypair *CredentialIssueOptions_Keypair @@ -2036,6 +2055,7 @@ func (b0 CredentialIssueOptions_builder) Build() *CredentialIssueOptions { m0 := &CredentialIssueOptions{} b, x := &b0, m0 _, _ = b, x + x.xxx_hidden_SecretResourceTypeId = b.SecretResourceTypeId if b.ApiKey != nil { x.xxx_hidden_Options = &credentialIssueOptions_ApiKey_{b.ApiKey} } @@ -5819,8 +5839,11 @@ func (b0 EncryptionConfig_JWKPublicKeyConfig_builder) Build() *EncryptionConfig_ // supported by the configured age provider. EncryptedData.encrypted_bytes // contains a standard binary age file when this config is used. The provider // sets EncryptedData.key_ids to one lowercase hexadecimal SHA-256 digest of -// the UTF-8 canonical recipient string. It leaves the deprecated -// EncryptedData.key_id empty. +// the UTF-8 canonical recipient string, and leaves the deprecated +// EncryptedData.key_id empty. Rather than reimplement this derivation, +// consumers written in Go should call +// pkg/crypto/providers/age.KeyIDForRecipient, which is the single source of +// truth for the convention. type EncryptionConfig_AgeRecipientConfig struct { state protoimpl.MessageState `protogen:"opaque.v1"` xxx_hidden_Recipient string `protobuf:"bytes,1,opt,name=recipient,proto3"` @@ -5980,8 +6003,9 @@ const file_c1_connector_v2_resource_proto_rawDesc = "" + "\x10rsa_modulus_bits\x18\x02 \x01(\rH\x00R\x0ersaModulusBits\x12\x1b\n" + "\x03crv\x18\x03 \x01(\tB\a\xfaB\x04r\x02(@H\x00R\x03crvB\f\n" + "\n" + - "parameters\"\x9c\x04\n" + - "\x16CredentialIssueOptions\x12I\n" + + "parameters\"\xd3\x04\n" + + "\x16CredentialIssueOptions\x125\n" + + "\x17secret_resource_type_id\x18\x01 \x01(\tR\x14secretResourceTypeId\x12I\n" + "\aapi_key\x18d \x01(\v2..c1.connector.v2.CredentialIssueOptions.ApiKeyH\x00R\x06apiKey\x12K\n" + "\akeypair\x18e \x01(\v2/.c1.connector.v2.CredentialIssueOptions.KeypairH\x00R\akeypair\x12E\n" + "\x05token\x18f \x01(\v2-.c1.connector.v2.CredentialIssueOptions.TokenH\x00R\x05token\x12[\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.go index fe7c7788..f2606742 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.go @@ -104,7 +104,6 @@ type Task struct { // *Task_ActionGetSchema // *Task_ActionInvoke // *Task_ActionStatus - // *Task_CreateSyncDiff // *Task_CompactSyncs_ // *Task_ListEventFeeds // *Task_ListEvents @@ -332,15 +331,6 @@ func (x *Task) GetActionStatus() *Task_ActionStatusTask { return nil } -func (x *Task) GetCreateSyncDiff() *Task_CreateSyncDiffTask { - if x != nil { - if x, ok := x.TaskType.(*Task_CreateSyncDiff); ok { - return x.CreateSyncDiff - } - } - return nil -} - func (x *Task) GetCompactSyncs() *Task_CompactSyncs { if x != nil { if x, ok := x.TaskType.(*Task_CompactSyncs_); ok { @@ -544,14 +534,6 @@ func (x *Task) SetActionStatus(v *Task_ActionStatusTask) { x.TaskType = &Task_ActionStatus{v} } -func (x *Task) SetCreateSyncDiff(v *Task_CreateSyncDiffTask) { - if v == nil { - x.TaskType = nil - return - } - x.TaskType = &Task_CreateSyncDiff{v} -} - func (x *Task) SetCompactSyncs(v *Task_CompactSyncs) { if v == nil { x.TaskType = nil @@ -747,14 +729,6 @@ func (x *Task) HasActionStatus() bool { return ok } -func (x *Task) HasCreateSyncDiff() bool { - if x == nil { - return false - } - _, ok := x.TaskType.(*Task_CreateSyncDiff) - return ok -} - func (x *Task) HasCompactSyncs() bool { if x == nil { return false @@ -905,12 +879,6 @@ func (x *Task) ClearActionStatus() { } } -func (x *Task) ClearCreateSyncDiff() { - if _, ok := x.TaskType.(*Task_CreateSyncDiff); ok { - x.TaskType = nil - } -} - func (x *Task) ClearCompactSyncs() { if _, ok := x.TaskType.(*Task_CompactSyncs_); ok { x.TaskType = nil @@ -955,7 +923,6 @@ const Task_ActionListSchemas_case case_Task_TaskType = 115 const Task_ActionGetSchema_case case_Task_TaskType = 116 const Task_ActionInvoke_case case_Task_TaskType = 117 const Task_ActionStatus_case case_Task_TaskType = 118 -const Task_CreateSyncDiff_case case_Task_TaskType = 119 const Task_CompactSyncs_case case_Task_TaskType = 120 const Task_ListEventFeeds_case case_Task_TaskType = 121 const Task_ListEvents_case case_Task_TaskType = 122 @@ -1004,8 +971,6 @@ func (x *Task) WhichTaskType() case_Task_TaskType { return Task_ActionInvoke_case case *Task_ActionStatus: return Task_ActionStatus_case - case *Task_CreateSyncDiff: - return Task_CreateSyncDiff_case case *Task_CompactSyncs_: return Task_CompactSyncs_case case *Task_ListEventFeeds: @@ -1044,7 +1009,6 @@ type Task_builder struct { ActionGetSchema *Task_ActionGetSchemaTask ActionInvoke *Task_ActionInvokeTask ActionStatus *Task_ActionStatusTask - CreateSyncDiff *Task_CreateSyncDiffTask CompactSyncs *Task_CompactSyncs ListEventFeeds *Task_ListEventFeedsTask ListEvents *Task_ListEventsTask @@ -1116,9 +1080,6 @@ func (b0 Task_builder) Build() *Task { if b.ActionStatus != nil { x.TaskType = &Task_ActionStatus{b.ActionStatus} } - if b.CreateSyncDiff != nil { - x.TaskType = &Task_CreateSyncDiff{b.CreateSyncDiff} - } if b.CompactSyncs != nil { x.TaskType = &Task_CompactSyncs_{b.CompactSyncs} } @@ -1225,10 +1186,6 @@ type Task_ActionStatus struct { ActionStatus *Task_ActionStatusTask `protobuf:"bytes,118,opt,name=action_status,json=actionStatus,proto3,oneof"` } -type Task_CreateSyncDiff struct { - CreateSyncDiff *Task_CreateSyncDiffTask `protobuf:"bytes,119,opt,name=create_sync_diff,json=createSyncDiff,proto3,oneof"` -} - type Task_CompactSyncs_ struct { CompactSyncs *Task_CompactSyncs `protobuf:"bytes,120,opt,name=compact_syncs,json=compactSyncs,proto3,oneof"` } @@ -1283,8 +1240,6 @@ func (*Task_ActionInvoke) isTask_TaskType() {} func (*Task_ActionStatus) isTask_TaskType() {} -func (*Task_CreateSyncDiff) isTask_TaskType() {} - func (*Task_CompactSyncs_) isTask_TaskType() {} func (*Task_ListEventFeeds) isTask_TaskType() {} @@ -2935,7 +2890,7 @@ type Task_SyncFullTask struct { SyncResourceTypeIds []string `protobuf:"bytes,5,rep,name=sync_resource_type_ids,json=syncResourceTypeIds,proto3" json:"sync_resource_type_ids,omitempty"` // If true, skip syncing grants. Resources and entitlements will still be synced. SkipGrants bool `protobuf:"varint,6,opt,name=skip_grants,json=skipGrants,proto3" json:"skip_grants,omitempty"` - // Storage engine to use for the sync. If empty, the default engine will be used (currently SQLite). + // Storage engine to use for the sync. If empty, the default engine will be used (currently Pebble for new c1z files; existing files keep their on-disk format). StorageEngine string `protobuf:"bytes,7,opt,name=storage_engine,json=storageEngine,proto3" json:"storage_engine,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -3057,7 +3012,7 @@ type Task_SyncFullTask_builder struct { SyncResourceTypeIds []string // If true, skip syncing grants. Resources and entitlements will still be synced. SkipGrants bool - // Storage engine to use for the sync. If empty, the default engine will be used (currently SQLite). + // Storage engine to use for the sync. If empty, the default engine will be used (currently Pebble for new c1z files; existing files keep their on-disk format). StorageEngine string } @@ -4763,12 +4718,16 @@ func (b0 Task_ActionStatusTask_builder) Build() *Task_ActionStatusTask { return m0 } +// Deprecated: diff-sync support was removed from the SDK. The message +// is retained only to satisfy breaking-change detection; nothing +// produces or consumes it. +// +// Deprecated: Marked as deprecated in c1/connectorapi/baton/v1/baton.proto. type Task_CreateSyncDiffTask struct { - state protoimpl.MessageState `protogen:"hybrid.v1"` - // Open to suggestions here - BaseSyncId string `protobuf:"bytes,1,opt,name=base_sync_id,json=baseSyncId,proto3" json:"base_sync_id,omitempty"` - NewSyncId string `protobuf:"bytes,2,opt,name=new_sync_id,json=newSyncId,proto3" json:"new_sync_id,omitempty"` - Annotations []*anypb.Any `protobuf:"bytes,3,rep,name=annotations,proto3" json:"annotations,omitempty"` + state protoimpl.MessageState `protogen:"hybrid.v1"` + BaseSyncId string `protobuf:"bytes,1,opt,name=base_sync_id,json=baseSyncId,proto3" json:"base_sync_id,omitempty"` + NewSyncId string `protobuf:"bytes,2,opt,name=new_sync_id,json=newSyncId,proto3" json:"new_sync_id,omitempty"` + Annotations []*anypb.Any `protobuf:"bytes,3,rep,name=annotations,proto3" json:"annotations,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -4831,10 +4790,10 @@ func (x *Task_CreateSyncDiffTask) SetAnnotations(v []*anypb.Any) { x.Annotations = v } +// Deprecated: Marked as deprecated in c1/connectorapi/baton/v1/baton.proto. type Task_CreateSyncDiffTask_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - // Open to suggestions here BaseSyncId string NewSyncId string Annotations []*anypb.Any @@ -5639,7 +5598,7 @@ var File_c1_connectorapi_baton_v1_baton_proto protoreflect.FileDescriptor const file_c1_connectorapi_baton_v1_baton_proto_rawDesc = "" + "\n" + - "$c1/connectorapi/baton/v1/baton.proto\x12\x18c1.connectorapi.baton.v1\x1a\x1fc1/connector/v2/connector.proto\x1a!c1/connector/v2/entitlement.proto\x1a\x1bc1/connector/v2/grant.proto\x1a\x1ec1/connector/v2/resource.proto\x1a\x1cc1/connector/v2/ticket.proto\x1a\x19google/protobuf/any.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\x1a\x17validate/validate.proto\"\xd51\n" + + "$c1/connectorapi/baton/v1/baton.proto\x12\x18c1.connectorapi.baton.v1\x1a\x1fc1/connector/v2/connector.proto\x1a!c1/connector/v2/entitlement.proto\x1a\x1bc1/connector/v2/grant.proto\x1a\x1ec1/connector/v2/resource.proto\x1a\x1cc1/connector/v2/ticket.proto\x1a\x19google/protobuf/any.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\x1a\x17validate/validate.proto\"\x921\n" + "\x04Task\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12=\n" + "\x06status\x18\x02 \x01(\x0e2%.c1.connectorapi.baton.v1.Task.StatusR\x06status\x12=\n" + @@ -5663,8 +5622,7 @@ const file_c1_connectorapi_baton_v1_baton_proto_rawDesc = "" + "\x13action_list_schemas\x18s \x01(\v24.c1.connectorapi.baton.v1.Task.ActionListSchemasTaskH\x00R\x11actionListSchemas\x12`\n" + "\x11action_get_schema\x18t \x01(\v22.c1.connectorapi.baton.v1.Task.ActionGetSchemaTaskH\x00R\x0factionGetSchema\x12V\n" + "\raction_invoke\x18u \x01(\v2/.c1.connectorapi.baton.v1.Task.ActionInvokeTaskH\x00R\factionInvoke\x12V\n" + - "\raction_status\x18v \x01(\v2/.c1.connectorapi.baton.v1.Task.ActionStatusTaskH\x00R\factionStatus\x12]\n" + - "\x10create_sync_diff\x18w \x01(\v21.c1.connectorapi.baton.v1.Task.CreateSyncDiffTaskH\x00R\x0ecreateSyncDiff\x12R\n" + + "\raction_status\x18v \x01(\v2/.c1.connectorapi.baton.v1.Task.ActionStatusTaskH\x00R\factionStatus\x12R\n" + "\rcompact_syncs\x18x \x01(\v2+.c1.connectorapi.baton.v1.Task.CompactSyncsH\x00R\fcompactSyncs\x12]\n" + "\x10list_event_feeds\x18y \x01(\v21.c1.connectorapi.baton.v1.Task.ListEventFeedsTaskH\x00R\x0elistEventFeeds\x12P\n" + "\vlist_events\x18z \x01(\v2-.c1.connectorapi.baton.v1.Task.ListEventsTaskH\x00R\n" + @@ -5756,12 +5714,12 @@ const file_c1_connectorapi_baton_v1_baton_proto_rawDesc = "" + "\x10ActionStatusTask\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x0e\n" + "\x02id\x18\x02 \x01(\tR\x02id\x126\n" + - "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x1a\x8e\x01\n" + + "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x1a\x92\x01\n" + "\x12CreateSyncDiffTask\x12 \n" + "\fbase_sync_id\x18\x01 \x01(\tR\n" + "baseSyncId\x12\x1e\n" + "\vnew_sync_id\x18\x02 \x01(\tR\tnewSyncId\x126\n" + - "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x1a\xf9\x01\n" + + "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations:\x02\x18\x01\x1a\xf9\x01\n" + "\fCompactSyncs\x12h\n" + "\x11compactable_syncs\x18\x01 \x03(\v2;.c1.connectorapi.baton.v1.Task.CompactSyncs.CompactableSyncR\x10compactableSyncs\x126\n" + "\vannotations\x18\x02 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x1aG\n" + @@ -5774,7 +5732,7 @@ const file_c1_connectorapi_baton_v1_baton_proto_rawDesc = "" + "\x10STATUS_SCHEDULED\x10\x02\x12\x12\n" + "\x0eSTATUS_RUNNING\x10\x03\x12\x13\n" + "\x0fSTATUS_FINISHED\x10\x04B\v\n" + - "\ttask_type\"\xc9\a\n" + + "\ttask_typeJ\x04\bw\x10xR\x10create_sync_diff\"\xc9\a\n" + "\x18BatonServiceHelloRequest\x12#\n" + "\ahost_id\x18\x01 \x01(\tB\n" + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\x06hostId\x122\n" + @@ -5968,104 +5926,103 @@ var file_c1_connectorapi_baton_v1_baton_proto_depIdxs = []int32{ 35, // 17: c1.connectorapi.baton.v1.Task.action_get_schema:type_name -> c1.connectorapi.baton.v1.Task.ActionGetSchemaTask 36, // 18: c1.connectorapi.baton.v1.Task.action_invoke:type_name -> c1.connectorapi.baton.v1.Task.ActionInvokeTask 37, // 19: c1.connectorapi.baton.v1.Task.action_status:type_name -> c1.connectorapi.baton.v1.Task.ActionStatusTask - 38, // 20: c1.connectorapi.baton.v1.Task.create_sync_diff:type_name -> c1.connectorapi.baton.v1.Task.CreateSyncDiffTask - 39, // 21: c1.connectorapi.baton.v1.Task.compact_syncs:type_name -> c1.connectorapi.baton.v1.Task.CompactSyncs - 21, // 22: c1.connectorapi.baton.v1.Task.list_event_feeds:type_name -> c1.connectorapi.baton.v1.Task.ListEventFeedsTask - 20, // 23: c1.connectorapi.baton.v1.Task.list_events:type_name -> c1.connectorapi.baton.v1.Task.ListEventsTask - 28, // 24: c1.connectorapi.baton.v1.Task.issue_credential:type_name -> c1.connectorapi.baton.v1.Task.IssueCredentialTask - 41, // 25: c1.connectorapi.baton.v1.BatonServiceHelloRequest.build_info:type_name -> c1.connectorapi.baton.v1.BatonServiceHelloRequest.BuildInfo - 42, // 26: c1.connectorapi.baton.v1.BatonServiceHelloRequest.os_info:type_name -> c1.connectorapi.baton.v1.BatonServiceHelloRequest.OSInfo - 48, // 27: c1.connectorapi.baton.v1.BatonServiceHelloRequest.connector_metadata:type_name -> c1.connector.v2.ConnectorMetadata - 49, // 28: c1.connectorapi.baton.v1.BatonServiceHelloRequest.annotations:type_name -> google.protobuf.Any - 49, // 29: c1.connectorapi.baton.v1.BatonServiceHelloResponse.annotations:type_name -> google.protobuf.Any - 49, // 30: c1.connectorapi.baton.v1.BatonServiceGetTasksRequest.annotations:type_name -> google.protobuf.Any - 1, // 31: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.tasks:type_name -> c1.connectorapi.baton.v1.Task - 50, // 32: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.next_poll:type_name -> google.protobuf.Duration - 50, // 33: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.next_heartbeat:type_name -> google.protobuf.Duration - 49, // 34: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.annotations:type_name -> google.protobuf.Any - 1, // 35: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.task:type_name -> c1.connectorapi.baton.v1.Task - 50, // 36: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.next_poll:type_name -> google.protobuf.Duration - 50, // 37: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.next_heartbeat:type_name -> google.protobuf.Duration - 49, // 38: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.annotations:type_name -> google.protobuf.Any - 49, // 39: c1.connectorapi.baton.v1.BatonServiceHeartbeatRequest.annotations:type_name -> google.protobuf.Any - 50, // 40: c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse.next_heartbeat:type_name -> google.protobuf.Duration - 49, // 41: c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse.annotations:type_name -> google.protobuf.Any - 43, // 42: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.metadata:type_name -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadMetadata - 44, // 43: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.data:type_name -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadData - 45, // 44: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.eof:type_name -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadEOF - 49, // 45: c1.connectorapi.baton.v1.BatonServiceUploadAssetResponse.annotations:type_name -> google.protobuf.Any - 51, // 46: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.status:type_name -> google.rpc.Status - 46, // 47: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.error:type_name -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error - 47, // 48: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.success:type_name -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success - 49, // 49: c1.connectorapi.baton.v1.BatonServiceFinishTaskResponse.annotations:type_name -> google.protobuf.Any - 49, // 50: c1.connectorapi.baton.v1.Task.NoneTask.annotations:type_name -> google.protobuf.Any - 49, // 51: c1.connectorapi.baton.v1.Task.HelloTask.annotations:type_name -> google.protobuf.Any - 49, // 52: c1.connectorapi.baton.v1.Task.SyncFullTask.annotations:type_name -> google.protobuf.Any - 52, // 53: c1.connectorapi.baton.v1.Task.SyncFullTask.targeted_sync_resources:type_name -> c1.connector.v2.Resource - 49, // 54: c1.connectorapi.baton.v1.Task.EventFeedTask.annotations:type_name -> google.protobuf.Any - 53, // 55: c1.connectorapi.baton.v1.Task.EventFeedTask.start_at:type_name -> google.protobuf.Timestamp - 49, // 56: c1.connectorapi.baton.v1.Task.ListEventsTask.annotations:type_name -> google.protobuf.Any - 53, // 57: c1.connectorapi.baton.v1.Task.ListEventsTask.start_at:type_name -> google.protobuf.Timestamp - 49, // 58: c1.connectorapi.baton.v1.Task.ListEventFeedsTask.annotations:type_name -> google.protobuf.Any - 54, // 59: c1.connectorapi.baton.v1.Task.GrantTask.entitlement:type_name -> c1.connector.v2.Entitlement - 52, // 60: c1.connectorapi.baton.v1.Task.GrantTask.principal:type_name -> c1.connector.v2.Resource - 49, // 61: c1.connectorapi.baton.v1.Task.GrantTask.annotations:type_name -> google.protobuf.Any - 50, // 62: c1.connectorapi.baton.v1.Task.GrantTask.duration:type_name -> google.protobuf.Duration - 55, // 63: c1.connectorapi.baton.v1.Task.RevokeTask.grant:type_name -> c1.connector.v2.Grant - 49, // 64: c1.connectorapi.baton.v1.Task.RevokeTask.annotations:type_name -> google.protobuf.Any - 56, // 65: c1.connectorapi.baton.v1.Task.CreateAccountTask.account_info:type_name -> c1.connector.v2.AccountInfo - 57, // 66: c1.connectorapi.baton.v1.Task.CreateAccountTask.credential_options:type_name -> c1.connector.v2.CredentialOptions - 58, // 67: c1.connectorapi.baton.v1.Task.CreateAccountTask.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig - 52, // 68: c1.connectorapi.baton.v1.Task.CreateResourceTask.resource:type_name -> c1.connector.v2.Resource - 59, // 69: c1.connectorapi.baton.v1.Task.DeleteResourceTask.resource_id:type_name -> c1.connector.v2.ResourceId - 59, // 70: c1.connectorapi.baton.v1.Task.DeleteResourceTask.parent_resource_id:type_name -> c1.connector.v2.ResourceId - 59, // 71: c1.connectorapi.baton.v1.Task.RotateCredentialsTask.resource_id:type_name -> c1.connector.v2.ResourceId - 57, // 72: c1.connectorapi.baton.v1.Task.RotateCredentialsTask.credential_options:type_name -> c1.connector.v2.CredentialOptions - 58, // 73: c1.connectorapi.baton.v1.Task.RotateCredentialsTask.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig - 59, // 74: c1.connectorapi.baton.v1.Task.IssueCredentialTask.identity_id:type_name -> c1.connector.v2.ResourceId - 60, // 75: c1.connectorapi.baton.v1.Task.IssueCredentialTask.credential_options:type_name -> c1.connector.v2.CredentialIssueOptions - 58, // 76: c1.connectorapi.baton.v1.Task.IssueCredentialTask.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig - 53, // 77: c1.connectorapi.baton.v1.Task.IssueCredentialTask.expires_at:type_name -> google.protobuf.Timestamp - 61, // 78: c1.connectorapi.baton.v1.Task.CreateTicketTask.ticket_request:type_name -> c1.connector.v2.TicketRequest - 62, // 79: c1.connectorapi.baton.v1.Task.CreateTicketTask.ticket_schema:type_name -> c1.connector.v2.TicketSchema - 49, // 80: c1.connectorapi.baton.v1.Task.CreateTicketTask.annotations:type_name -> google.protobuf.Any - 29, // 81: c1.connectorapi.baton.v1.Task.BulkCreateTicketsTask.ticket_requests:type_name -> c1.connectorapi.baton.v1.Task.CreateTicketTask - 33, // 82: c1.connectorapi.baton.v1.Task.BulkGetTicketsTask.ticket_requests:type_name -> c1.connectorapi.baton.v1.Task.GetTicketTask - 49, // 83: c1.connectorapi.baton.v1.Task.ListTicketSchemasTask.annotations:type_name -> google.protobuf.Any - 49, // 84: c1.connectorapi.baton.v1.Task.GetTicketTask.annotations:type_name -> google.protobuf.Any - 49, // 85: c1.connectorapi.baton.v1.Task.ActionListSchemasTask.annotations:type_name -> google.protobuf.Any - 49, // 86: c1.connectorapi.baton.v1.Task.ActionGetSchemaTask.annotations:type_name -> google.protobuf.Any - 63, // 87: c1.connectorapi.baton.v1.Task.ActionInvokeTask.args:type_name -> google.protobuf.Struct - 49, // 88: c1.connectorapi.baton.v1.Task.ActionInvokeTask.annotations:type_name -> google.protobuf.Any - 49, // 89: c1.connectorapi.baton.v1.Task.ActionStatusTask.annotations:type_name -> google.protobuf.Any - 49, // 90: c1.connectorapi.baton.v1.Task.CreateSyncDiffTask.annotations:type_name -> google.protobuf.Any - 40, // 91: c1.connectorapi.baton.v1.Task.CompactSyncs.compactable_syncs:type_name -> c1.connectorapi.baton.v1.Task.CompactSyncs.CompactableSync - 49, // 92: c1.connectorapi.baton.v1.Task.CompactSyncs.annotations:type_name -> google.protobuf.Any - 49, // 93: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadMetadata.annotations:type_name -> google.protobuf.Any - 49, // 94: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadEOF.annotations:type_name -> google.protobuf.Any - 49, // 95: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.annotations:type_name -> google.protobuf.Any - 49, // 96: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.response:type_name -> google.protobuf.Any - 49, // 97: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.annotations:type_name -> google.protobuf.Any - 49, // 98: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.response:type_name -> google.protobuf.Any - 2, // 99: c1.connectorapi.baton.v1.BatonService.Hello:input_type -> c1.connectorapi.baton.v1.BatonServiceHelloRequest - 4, // 100: c1.connectorapi.baton.v1.BatonService.GetTask:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskRequest - 5, // 101: c1.connectorapi.baton.v1.BatonService.GetTasks:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksRequest - 8, // 102: c1.connectorapi.baton.v1.BatonService.Heartbeat:input_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatRequest - 12, // 103: c1.connectorapi.baton.v1.BatonService.FinishTask:input_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest - 10, // 104: c1.connectorapi.baton.v1.BatonService.UploadAsset:input_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest - 14, // 105: c1.connectorapi.baton.v1.BatonService.StartDebugging:input_type -> c1.connectorapi.baton.v1.StartDebuggingRequest - 3, // 106: c1.connectorapi.baton.v1.BatonService.Hello:output_type -> c1.connectorapi.baton.v1.BatonServiceHelloResponse - 7, // 107: c1.connectorapi.baton.v1.BatonService.GetTask:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskResponse - 6, // 108: c1.connectorapi.baton.v1.BatonService.GetTasks:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksResponse - 9, // 109: c1.connectorapi.baton.v1.BatonService.Heartbeat:output_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse - 13, // 110: c1.connectorapi.baton.v1.BatonService.FinishTask:output_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskResponse - 11, // 111: c1.connectorapi.baton.v1.BatonService.UploadAsset:output_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetResponse - 15, // 112: c1.connectorapi.baton.v1.BatonService.StartDebugging:output_type -> c1.connectorapi.baton.v1.StartDebuggingResponse - 106, // [106:113] is the sub-list for method output_type - 99, // [99:106] is the sub-list for method input_type - 99, // [99:99] is the sub-list for extension type_name - 99, // [99:99] is the sub-list for extension extendee - 0, // [0:99] is the sub-list for field type_name + 39, // 20: c1.connectorapi.baton.v1.Task.compact_syncs:type_name -> c1.connectorapi.baton.v1.Task.CompactSyncs + 21, // 21: c1.connectorapi.baton.v1.Task.list_event_feeds:type_name -> c1.connectorapi.baton.v1.Task.ListEventFeedsTask + 20, // 22: c1.connectorapi.baton.v1.Task.list_events:type_name -> c1.connectorapi.baton.v1.Task.ListEventsTask + 28, // 23: c1.connectorapi.baton.v1.Task.issue_credential:type_name -> c1.connectorapi.baton.v1.Task.IssueCredentialTask + 41, // 24: c1.connectorapi.baton.v1.BatonServiceHelloRequest.build_info:type_name -> c1.connectorapi.baton.v1.BatonServiceHelloRequest.BuildInfo + 42, // 25: c1.connectorapi.baton.v1.BatonServiceHelloRequest.os_info:type_name -> c1.connectorapi.baton.v1.BatonServiceHelloRequest.OSInfo + 48, // 26: c1.connectorapi.baton.v1.BatonServiceHelloRequest.connector_metadata:type_name -> c1.connector.v2.ConnectorMetadata + 49, // 27: c1.connectorapi.baton.v1.BatonServiceHelloRequest.annotations:type_name -> google.protobuf.Any + 49, // 28: c1.connectorapi.baton.v1.BatonServiceHelloResponse.annotations:type_name -> google.protobuf.Any + 49, // 29: c1.connectorapi.baton.v1.BatonServiceGetTasksRequest.annotations:type_name -> google.protobuf.Any + 1, // 30: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.tasks:type_name -> c1.connectorapi.baton.v1.Task + 50, // 31: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.next_poll:type_name -> google.protobuf.Duration + 50, // 32: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.next_heartbeat:type_name -> google.protobuf.Duration + 49, // 33: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.annotations:type_name -> google.protobuf.Any + 1, // 34: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.task:type_name -> c1.connectorapi.baton.v1.Task + 50, // 35: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.next_poll:type_name -> google.protobuf.Duration + 50, // 36: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.next_heartbeat:type_name -> google.protobuf.Duration + 49, // 37: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.annotations:type_name -> google.protobuf.Any + 49, // 38: c1.connectorapi.baton.v1.BatonServiceHeartbeatRequest.annotations:type_name -> google.protobuf.Any + 50, // 39: c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse.next_heartbeat:type_name -> google.protobuf.Duration + 49, // 40: c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse.annotations:type_name -> google.protobuf.Any + 43, // 41: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.metadata:type_name -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadMetadata + 44, // 42: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.data:type_name -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadData + 45, // 43: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.eof:type_name -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadEOF + 49, // 44: c1.connectorapi.baton.v1.BatonServiceUploadAssetResponse.annotations:type_name -> google.protobuf.Any + 51, // 45: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.status:type_name -> google.rpc.Status + 46, // 46: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.error:type_name -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error + 47, // 47: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.success:type_name -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success + 49, // 48: c1.connectorapi.baton.v1.BatonServiceFinishTaskResponse.annotations:type_name -> google.protobuf.Any + 49, // 49: c1.connectorapi.baton.v1.Task.NoneTask.annotations:type_name -> google.protobuf.Any + 49, // 50: c1.connectorapi.baton.v1.Task.HelloTask.annotations:type_name -> google.protobuf.Any + 49, // 51: c1.connectorapi.baton.v1.Task.SyncFullTask.annotations:type_name -> google.protobuf.Any + 52, // 52: c1.connectorapi.baton.v1.Task.SyncFullTask.targeted_sync_resources:type_name -> c1.connector.v2.Resource + 49, // 53: c1.connectorapi.baton.v1.Task.EventFeedTask.annotations:type_name -> google.protobuf.Any + 53, // 54: c1.connectorapi.baton.v1.Task.EventFeedTask.start_at:type_name -> google.protobuf.Timestamp + 49, // 55: c1.connectorapi.baton.v1.Task.ListEventsTask.annotations:type_name -> google.protobuf.Any + 53, // 56: c1.connectorapi.baton.v1.Task.ListEventsTask.start_at:type_name -> google.protobuf.Timestamp + 49, // 57: c1.connectorapi.baton.v1.Task.ListEventFeedsTask.annotations:type_name -> google.protobuf.Any + 54, // 58: c1.connectorapi.baton.v1.Task.GrantTask.entitlement:type_name -> c1.connector.v2.Entitlement + 52, // 59: c1.connectorapi.baton.v1.Task.GrantTask.principal:type_name -> c1.connector.v2.Resource + 49, // 60: c1.connectorapi.baton.v1.Task.GrantTask.annotations:type_name -> google.protobuf.Any + 50, // 61: c1.connectorapi.baton.v1.Task.GrantTask.duration:type_name -> google.protobuf.Duration + 55, // 62: c1.connectorapi.baton.v1.Task.RevokeTask.grant:type_name -> c1.connector.v2.Grant + 49, // 63: c1.connectorapi.baton.v1.Task.RevokeTask.annotations:type_name -> google.protobuf.Any + 56, // 64: c1.connectorapi.baton.v1.Task.CreateAccountTask.account_info:type_name -> c1.connector.v2.AccountInfo + 57, // 65: c1.connectorapi.baton.v1.Task.CreateAccountTask.credential_options:type_name -> c1.connector.v2.CredentialOptions + 58, // 66: c1.connectorapi.baton.v1.Task.CreateAccountTask.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig + 52, // 67: c1.connectorapi.baton.v1.Task.CreateResourceTask.resource:type_name -> c1.connector.v2.Resource + 59, // 68: c1.connectorapi.baton.v1.Task.DeleteResourceTask.resource_id:type_name -> c1.connector.v2.ResourceId + 59, // 69: c1.connectorapi.baton.v1.Task.DeleteResourceTask.parent_resource_id:type_name -> c1.connector.v2.ResourceId + 59, // 70: c1.connectorapi.baton.v1.Task.RotateCredentialsTask.resource_id:type_name -> c1.connector.v2.ResourceId + 57, // 71: c1.connectorapi.baton.v1.Task.RotateCredentialsTask.credential_options:type_name -> c1.connector.v2.CredentialOptions + 58, // 72: c1.connectorapi.baton.v1.Task.RotateCredentialsTask.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig + 59, // 73: c1.connectorapi.baton.v1.Task.IssueCredentialTask.identity_id:type_name -> c1.connector.v2.ResourceId + 60, // 74: c1.connectorapi.baton.v1.Task.IssueCredentialTask.credential_options:type_name -> c1.connector.v2.CredentialIssueOptions + 58, // 75: c1.connectorapi.baton.v1.Task.IssueCredentialTask.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig + 53, // 76: c1.connectorapi.baton.v1.Task.IssueCredentialTask.expires_at:type_name -> google.protobuf.Timestamp + 61, // 77: c1.connectorapi.baton.v1.Task.CreateTicketTask.ticket_request:type_name -> c1.connector.v2.TicketRequest + 62, // 78: c1.connectorapi.baton.v1.Task.CreateTicketTask.ticket_schema:type_name -> c1.connector.v2.TicketSchema + 49, // 79: c1.connectorapi.baton.v1.Task.CreateTicketTask.annotations:type_name -> google.protobuf.Any + 29, // 80: c1.connectorapi.baton.v1.Task.BulkCreateTicketsTask.ticket_requests:type_name -> c1.connectorapi.baton.v1.Task.CreateTicketTask + 33, // 81: c1.connectorapi.baton.v1.Task.BulkGetTicketsTask.ticket_requests:type_name -> c1.connectorapi.baton.v1.Task.GetTicketTask + 49, // 82: c1.connectorapi.baton.v1.Task.ListTicketSchemasTask.annotations:type_name -> google.protobuf.Any + 49, // 83: c1.connectorapi.baton.v1.Task.GetTicketTask.annotations:type_name -> google.protobuf.Any + 49, // 84: c1.connectorapi.baton.v1.Task.ActionListSchemasTask.annotations:type_name -> google.protobuf.Any + 49, // 85: c1.connectorapi.baton.v1.Task.ActionGetSchemaTask.annotations:type_name -> google.protobuf.Any + 63, // 86: c1.connectorapi.baton.v1.Task.ActionInvokeTask.args:type_name -> google.protobuf.Struct + 49, // 87: c1.connectorapi.baton.v1.Task.ActionInvokeTask.annotations:type_name -> google.protobuf.Any + 49, // 88: c1.connectorapi.baton.v1.Task.ActionStatusTask.annotations:type_name -> google.protobuf.Any + 49, // 89: c1.connectorapi.baton.v1.Task.CreateSyncDiffTask.annotations:type_name -> google.protobuf.Any + 40, // 90: c1.connectorapi.baton.v1.Task.CompactSyncs.compactable_syncs:type_name -> c1.connectorapi.baton.v1.Task.CompactSyncs.CompactableSync + 49, // 91: c1.connectorapi.baton.v1.Task.CompactSyncs.annotations:type_name -> google.protobuf.Any + 49, // 92: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadMetadata.annotations:type_name -> google.protobuf.Any + 49, // 93: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadEOF.annotations:type_name -> google.protobuf.Any + 49, // 94: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.annotations:type_name -> google.protobuf.Any + 49, // 95: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.response:type_name -> google.protobuf.Any + 49, // 96: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.annotations:type_name -> google.protobuf.Any + 49, // 97: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.response:type_name -> google.protobuf.Any + 2, // 98: c1.connectorapi.baton.v1.BatonService.Hello:input_type -> c1.connectorapi.baton.v1.BatonServiceHelloRequest + 4, // 99: c1.connectorapi.baton.v1.BatonService.GetTask:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskRequest + 5, // 100: c1.connectorapi.baton.v1.BatonService.GetTasks:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksRequest + 8, // 101: c1.connectorapi.baton.v1.BatonService.Heartbeat:input_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatRequest + 12, // 102: c1.connectorapi.baton.v1.BatonService.FinishTask:input_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest + 10, // 103: c1.connectorapi.baton.v1.BatonService.UploadAsset:input_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest + 14, // 104: c1.connectorapi.baton.v1.BatonService.StartDebugging:input_type -> c1.connectorapi.baton.v1.StartDebuggingRequest + 3, // 105: c1.connectorapi.baton.v1.BatonService.Hello:output_type -> c1.connectorapi.baton.v1.BatonServiceHelloResponse + 7, // 106: c1.connectorapi.baton.v1.BatonService.GetTask:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskResponse + 6, // 107: c1.connectorapi.baton.v1.BatonService.GetTasks:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksResponse + 9, // 108: c1.connectorapi.baton.v1.BatonService.Heartbeat:output_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse + 13, // 109: c1.connectorapi.baton.v1.BatonService.FinishTask:output_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskResponse + 11, // 110: c1.connectorapi.baton.v1.BatonService.UploadAsset:output_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetResponse + 15, // 111: c1.connectorapi.baton.v1.BatonService.StartDebugging:output_type -> c1.connectorapi.baton.v1.StartDebuggingResponse + 105, // [105:112] is the sub-list for method output_type + 98, // [98:105] is the sub-list for method input_type + 98, // [98:98] is the sub-list for extension type_name + 98, // [98:98] is the sub-list for extension extendee + 0, // [0:98] is the sub-list for field type_name } func init() { file_c1_connectorapi_baton_v1_baton_proto_init() } @@ -6093,7 +6050,6 @@ func file_c1_connectorapi_baton_v1_baton_proto_init() { (*Task_ActionGetSchema)(nil), (*Task_ActionInvoke)(nil), (*Task_ActionStatus)(nil), - (*Task_CreateSyncDiff)(nil), (*Task_CompactSyncs_)(nil), (*Task_ListEventFeeds)(nil), (*Task_ListEvents)(nil), diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.validate.go index 559b0b0d..a333caa7 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.validate.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton.pb.validate.go @@ -842,47 +842,6 @@ func (m *Task) validate(all bool) error { } } - case *Task_CreateSyncDiff: - if v == nil { - err := TaskValidationError{ - field: "TaskType", - reason: "oneof value cannot be a typed-nil", - } - if !all { - return err - } - errors = append(errors, err) - } - - if all { - switch v := interface{}(m.GetCreateSyncDiff()).(type) { - case interface{ ValidateAll() error }: - if err := v.ValidateAll(); err != nil { - errors = append(errors, TaskValidationError{ - field: "CreateSyncDiff", - reason: "embedded message failed validation", - cause: err, - }) - } - case interface{ Validate() error }: - if err := v.Validate(); err != nil { - errors = append(errors, TaskValidationError{ - field: "CreateSyncDiff", - reason: "embedded message failed validation", - cause: err, - }) - } - } - } else if v, ok := interface{}(m.GetCreateSyncDiff()).(interface{ Validate() error }); ok { - if err := v.Validate(); err != nil { - return TaskValidationError{ - field: "CreateSyncDiff", - reason: "embedded message failed validation", - cause: err, - } - } - } - case *Task_CompactSyncs_: if v == nil { err := TaskValidationError{ diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton_protoopaque.pb.go index 833a47a4..6a985011 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1/baton_protoopaque.pb.go @@ -299,15 +299,6 @@ func (x *Task) GetActionStatus() *Task_ActionStatusTask { return nil } -func (x *Task) GetCreateSyncDiff() *Task_CreateSyncDiffTask { - if x != nil { - if x, ok := x.xxx_hidden_TaskType.(*task_CreateSyncDiff); ok { - return x.CreateSyncDiff - } - } - return nil -} - func (x *Task) GetCompactSyncs() *Task_CompactSyncs { if x != nil { if x, ok := x.xxx_hidden_TaskType.(*task_CompactSyncs_); ok { @@ -511,14 +502,6 @@ func (x *Task) SetActionStatus(v *Task_ActionStatusTask) { x.xxx_hidden_TaskType = &task_ActionStatus{v} } -func (x *Task) SetCreateSyncDiff(v *Task_CreateSyncDiffTask) { - if v == nil { - x.xxx_hidden_TaskType = nil - return - } - x.xxx_hidden_TaskType = &task_CreateSyncDiff{v} -} - func (x *Task) SetCompactSyncs(v *Task_CompactSyncs) { if v == nil { x.xxx_hidden_TaskType = nil @@ -714,14 +697,6 @@ func (x *Task) HasActionStatus() bool { return ok } -func (x *Task) HasCreateSyncDiff() bool { - if x == nil { - return false - } - _, ok := x.xxx_hidden_TaskType.(*task_CreateSyncDiff) - return ok -} - func (x *Task) HasCompactSyncs() bool { if x == nil { return false @@ -872,12 +847,6 @@ func (x *Task) ClearActionStatus() { } } -func (x *Task) ClearCreateSyncDiff() { - if _, ok := x.xxx_hidden_TaskType.(*task_CreateSyncDiff); ok { - x.xxx_hidden_TaskType = nil - } -} - func (x *Task) ClearCompactSyncs() { if _, ok := x.xxx_hidden_TaskType.(*task_CompactSyncs_); ok { x.xxx_hidden_TaskType = nil @@ -922,7 +891,6 @@ const Task_ActionListSchemas_case case_Task_TaskType = 115 const Task_ActionGetSchema_case case_Task_TaskType = 116 const Task_ActionInvoke_case case_Task_TaskType = 117 const Task_ActionStatus_case case_Task_TaskType = 118 -const Task_CreateSyncDiff_case case_Task_TaskType = 119 const Task_CompactSyncs_case case_Task_TaskType = 120 const Task_ListEventFeeds_case case_Task_TaskType = 121 const Task_ListEvents_case case_Task_TaskType = 122 @@ -971,8 +939,6 @@ func (x *Task) WhichTaskType() case_Task_TaskType { return Task_ActionInvoke_case case *task_ActionStatus: return Task_ActionStatus_case - case *task_CreateSyncDiff: - return Task_CreateSyncDiff_case case *task_CompactSyncs_: return Task_CompactSyncs_case case *task_ListEventFeeds: @@ -1011,7 +977,6 @@ type Task_builder struct { ActionGetSchema *Task_ActionGetSchemaTask ActionInvoke *Task_ActionInvokeTask ActionStatus *Task_ActionStatusTask - CreateSyncDiff *Task_CreateSyncDiffTask CompactSyncs *Task_CompactSyncs ListEventFeeds *Task_ListEventFeedsTask ListEvents *Task_ListEventsTask @@ -1083,9 +1048,6 @@ func (b0 Task_builder) Build() *Task { if b.ActionStatus != nil { x.xxx_hidden_TaskType = &task_ActionStatus{b.ActionStatus} } - if b.CreateSyncDiff != nil { - x.xxx_hidden_TaskType = &task_CreateSyncDiff{b.CreateSyncDiff} - } if b.CompactSyncs != nil { x.xxx_hidden_TaskType = &task_CompactSyncs_{b.CompactSyncs} } @@ -1192,10 +1154,6 @@ type task_ActionStatus struct { ActionStatus *Task_ActionStatusTask `protobuf:"bytes,118,opt,name=action_status,json=actionStatus,proto3,oneof"` } -type task_CreateSyncDiff struct { - CreateSyncDiff *Task_CreateSyncDiffTask `protobuf:"bytes,119,opt,name=create_sync_diff,json=createSyncDiff,proto3,oneof"` -} - type task_CompactSyncs_ struct { CompactSyncs *Task_CompactSyncs `protobuf:"bytes,120,opt,name=compact_syncs,json=compactSyncs,proto3,oneof"` } @@ -1250,8 +1208,6 @@ func (*task_ActionInvoke) isTask_TaskType() {} func (*task_ActionStatus) isTask_TaskType() {} -func (*task_CreateSyncDiff) isTask_TaskType() {} - func (*task_CompactSyncs_) isTask_TaskType() {} func (*task_ListEventFeeds) isTask_TaskType() {} @@ -3022,7 +2978,7 @@ type Task_SyncFullTask_builder struct { SyncResourceTypeIds []string // If true, skip syncing grants. Resources and entitlements will still be synced. SkipGrants bool - // Storage engine to use for the sync. If empty, the default engine will be used (currently SQLite). + // Storage engine to use for the sync. If empty, the default engine will be used (currently Pebble for new c1z files; existing files keep their on-disk format). StorageEngine string } @@ -4760,6 +4716,11 @@ func (b0 Task_ActionStatusTask_builder) Build() *Task_ActionStatusTask { return m0 } +// Deprecated: diff-sync support was removed from the SDK. The message +// is retained only to satisfy breaking-change detection; nothing +// produces or consumes it. +// +// Deprecated: Marked as deprecated in c1/connectorapi/baton/v1/baton.proto. type Task_CreateSyncDiffTask struct { state protoimpl.MessageState `protogen:"opaque.v1"` xxx_hidden_BaseSyncId string `protobuf:"bytes,1,opt,name=base_sync_id,json=baseSyncId,proto3"` @@ -4829,10 +4790,10 @@ func (x *Task_CreateSyncDiffTask) SetAnnotations(v []*anypb.Any) { x.xxx_hidden_Annotations = &v } +// Deprecated: Marked as deprecated in c1/connectorapi/baton/v1/baton.proto. type Task_CreateSyncDiffTask_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - // Open to suggestions here BaseSyncId string NewSyncId string Annotations []*anypb.Any @@ -5646,7 +5607,7 @@ var File_c1_connectorapi_baton_v1_baton_proto protoreflect.FileDescriptor const file_c1_connectorapi_baton_v1_baton_proto_rawDesc = "" + "\n" + - "$c1/connectorapi/baton/v1/baton.proto\x12\x18c1.connectorapi.baton.v1\x1a\x1fc1/connector/v2/connector.proto\x1a!c1/connector/v2/entitlement.proto\x1a\x1bc1/connector/v2/grant.proto\x1a\x1ec1/connector/v2/resource.proto\x1a\x1cc1/connector/v2/ticket.proto\x1a\x19google/protobuf/any.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\x1a\x17validate/validate.proto\"\xd51\n" + + "$c1/connectorapi/baton/v1/baton.proto\x12\x18c1.connectorapi.baton.v1\x1a\x1fc1/connector/v2/connector.proto\x1a!c1/connector/v2/entitlement.proto\x1a\x1bc1/connector/v2/grant.proto\x1a\x1ec1/connector/v2/resource.proto\x1a\x1cc1/connector/v2/ticket.proto\x1a\x19google/protobuf/any.proto\x1a\x1egoogle/protobuf/duration.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x17google/rpc/status.proto\x1a\x17validate/validate.proto\"\x921\n" + "\x04Task\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12=\n" + "\x06status\x18\x02 \x01(\x0e2%.c1.connectorapi.baton.v1.Task.StatusR\x06status\x12=\n" + @@ -5670,8 +5631,7 @@ const file_c1_connectorapi_baton_v1_baton_proto_rawDesc = "" + "\x13action_list_schemas\x18s \x01(\v24.c1.connectorapi.baton.v1.Task.ActionListSchemasTaskH\x00R\x11actionListSchemas\x12`\n" + "\x11action_get_schema\x18t \x01(\v22.c1.connectorapi.baton.v1.Task.ActionGetSchemaTaskH\x00R\x0factionGetSchema\x12V\n" + "\raction_invoke\x18u \x01(\v2/.c1.connectorapi.baton.v1.Task.ActionInvokeTaskH\x00R\factionInvoke\x12V\n" + - "\raction_status\x18v \x01(\v2/.c1.connectorapi.baton.v1.Task.ActionStatusTaskH\x00R\factionStatus\x12]\n" + - "\x10create_sync_diff\x18w \x01(\v21.c1.connectorapi.baton.v1.Task.CreateSyncDiffTaskH\x00R\x0ecreateSyncDiff\x12R\n" + + "\raction_status\x18v \x01(\v2/.c1.connectorapi.baton.v1.Task.ActionStatusTaskH\x00R\factionStatus\x12R\n" + "\rcompact_syncs\x18x \x01(\v2+.c1.connectorapi.baton.v1.Task.CompactSyncsH\x00R\fcompactSyncs\x12]\n" + "\x10list_event_feeds\x18y \x01(\v21.c1.connectorapi.baton.v1.Task.ListEventFeedsTaskH\x00R\x0elistEventFeeds\x12P\n" + "\vlist_events\x18z \x01(\v2-.c1.connectorapi.baton.v1.Task.ListEventsTaskH\x00R\n" + @@ -5763,12 +5723,12 @@ const file_c1_connectorapi_baton_v1_baton_proto_rawDesc = "" + "\x10ActionStatusTask\x12\x12\n" + "\x04name\x18\x01 \x01(\tR\x04name\x12\x0e\n" + "\x02id\x18\x02 \x01(\tR\x02id\x126\n" + - "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x1a\x8e\x01\n" + + "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x1a\x92\x01\n" + "\x12CreateSyncDiffTask\x12 \n" + "\fbase_sync_id\x18\x01 \x01(\tR\n" + "baseSyncId\x12\x1e\n" + "\vnew_sync_id\x18\x02 \x01(\tR\tnewSyncId\x126\n" + - "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x1a\xf9\x01\n" + + "\vannotations\x18\x03 \x03(\v2\x14.google.protobuf.AnyR\vannotations:\x02\x18\x01\x1a\xf9\x01\n" + "\fCompactSyncs\x12h\n" + "\x11compactable_syncs\x18\x01 \x03(\v2;.c1.connectorapi.baton.v1.Task.CompactSyncs.CompactableSyncR\x10compactableSyncs\x126\n" + "\vannotations\x18\x02 \x03(\v2\x14.google.protobuf.AnyR\vannotations\x1aG\n" + @@ -5781,7 +5741,7 @@ const file_c1_connectorapi_baton_v1_baton_proto_rawDesc = "" + "\x10STATUS_SCHEDULED\x10\x02\x12\x12\n" + "\x0eSTATUS_RUNNING\x10\x03\x12\x13\n" + "\x0fSTATUS_FINISHED\x10\x04B\v\n" + - "\ttask_type\"\xc9\a\n" + + "\ttask_typeJ\x04\bw\x10xR\x10create_sync_diff\"\xc9\a\n" + "\x18BatonServiceHelloRequest\x12#\n" + "\ahost_id\x18\x01 \x01(\tB\n" + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\x06hostId\x122\n" + @@ -5975,104 +5935,103 @@ var file_c1_connectorapi_baton_v1_baton_proto_depIdxs = []int32{ 35, // 17: c1.connectorapi.baton.v1.Task.action_get_schema:type_name -> c1.connectorapi.baton.v1.Task.ActionGetSchemaTask 36, // 18: c1.connectorapi.baton.v1.Task.action_invoke:type_name -> c1.connectorapi.baton.v1.Task.ActionInvokeTask 37, // 19: c1.connectorapi.baton.v1.Task.action_status:type_name -> c1.connectorapi.baton.v1.Task.ActionStatusTask - 38, // 20: c1.connectorapi.baton.v1.Task.create_sync_diff:type_name -> c1.connectorapi.baton.v1.Task.CreateSyncDiffTask - 39, // 21: c1.connectorapi.baton.v1.Task.compact_syncs:type_name -> c1.connectorapi.baton.v1.Task.CompactSyncs - 21, // 22: c1.connectorapi.baton.v1.Task.list_event_feeds:type_name -> c1.connectorapi.baton.v1.Task.ListEventFeedsTask - 20, // 23: c1.connectorapi.baton.v1.Task.list_events:type_name -> c1.connectorapi.baton.v1.Task.ListEventsTask - 28, // 24: c1.connectorapi.baton.v1.Task.issue_credential:type_name -> c1.connectorapi.baton.v1.Task.IssueCredentialTask - 41, // 25: c1.connectorapi.baton.v1.BatonServiceHelloRequest.build_info:type_name -> c1.connectorapi.baton.v1.BatonServiceHelloRequest.BuildInfo - 42, // 26: c1.connectorapi.baton.v1.BatonServiceHelloRequest.os_info:type_name -> c1.connectorapi.baton.v1.BatonServiceHelloRequest.OSInfo - 48, // 27: c1.connectorapi.baton.v1.BatonServiceHelloRequest.connector_metadata:type_name -> c1.connector.v2.ConnectorMetadata - 49, // 28: c1.connectorapi.baton.v1.BatonServiceHelloRequest.annotations:type_name -> google.protobuf.Any - 49, // 29: c1.connectorapi.baton.v1.BatonServiceHelloResponse.annotations:type_name -> google.protobuf.Any - 49, // 30: c1.connectorapi.baton.v1.BatonServiceGetTasksRequest.annotations:type_name -> google.protobuf.Any - 1, // 31: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.tasks:type_name -> c1.connectorapi.baton.v1.Task - 50, // 32: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.next_poll:type_name -> google.protobuf.Duration - 50, // 33: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.next_heartbeat:type_name -> google.protobuf.Duration - 49, // 34: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.annotations:type_name -> google.protobuf.Any - 1, // 35: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.task:type_name -> c1.connectorapi.baton.v1.Task - 50, // 36: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.next_poll:type_name -> google.protobuf.Duration - 50, // 37: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.next_heartbeat:type_name -> google.protobuf.Duration - 49, // 38: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.annotations:type_name -> google.protobuf.Any - 49, // 39: c1.connectorapi.baton.v1.BatonServiceHeartbeatRequest.annotations:type_name -> google.protobuf.Any - 50, // 40: c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse.next_heartbeat:type_name -> google.protobuf.Duration - 49, // 41: c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse.annotations:type_name -> google.protobuf.Any - 43, // 42: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.metadata:type_name -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadMetadata - 44, // 43: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.data:type_name -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadData - 45, // 44: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.eof:type_name -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadEOF - 49, // 45: c1.connectorapi.baton.v1.BatonServiceUploadAssetResponse.annotations:type_name -> google.protobuf.Any - 51, // 46: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.status:type_name -> google.rpc.Status - 46, // 47: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.error:type_name -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error - 47, // 48: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.success:type_name -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success - 49, // 49: c1.connectorapi.baton.v1.BatonServiceFinishTaskResponse.annotations:type_name -> google.protobuf.Any - 49, // 50: c1.connectorapi.baton.v1.Task.NoneTask.annotations:type_name -> google.protobuf.Any - 49, // 51: c1.connectorapi.baton.v1.Task.HelloTask.annotations:type_name -> google.protobuf.Any - 49, // 52: c1.connectorapi.baton.v1.Task.SyncFullTask.annotations:type_name -> google.protobuf.Any - 52, // 53: c1.connectorapi.baton.v1.Task.SyncFullTask.targeted_sync_resources:type_name -> c1.connector.v2.Resource - 49, // 54: c1.connectorapi.baton.v1.Task.EventFeedTask.annotations:type_name -> google.protobuf.Any - 53, // 55: c1.connectorapi.baton.v1.Task.EventFeedTask.start_at:type_name -> google.protobuf.Timestamp - 49, // 56: c1.connectorapi.baton.v1.Task.ListEventsTask.annotations:type_name -> google.protobuf.Any - 53, // 57: c1.connectorapi.baton.v1.Task.ListEventsTask.start_at:type_name -> google.protobuf.Timestamp - 49, // 58: c1.connectorapi.baton.v1.Task.ListEventFeedsTask.annotations:type_name -> google.protobuf.Any - 54, // 59: c1.connectorapi.baton.v1.Task.GrantTask.entitlement:type_name -> c1.connector.v2.Entitlement - 52, // 60: c1.connectorapi.baton.v1.Task.GrantTask.principal:type_name -> c1.connector.v2.Resource - 49, // 61: c1.connectorapi.baton.v1.Task.GrantTask.annotations:type_name -> google.protobuf.Any - 50, // 62: c1.connectorapi.baton.v1.Task.GrantTask.duration:type_name -> google.protobuf.Duration - 55, // 63: c1.connectorapi.baton.v1.Task.RevokeTask.grant:type_name -> c1.connector.v2.Grant - 49, // 64: c1.connectorapi.baton.v1.Task.RevokeTask.annotations:type_name -> google.protobuf.Any - 56, // 65: c1.connectorapi.baton.v1.Task.CreateAccountTask.account_info:type_name -> c1.connector.v2.AccountInfo - 57, // 66: c1.connectorapi.baton.v1.Task.CreateAccountTask.credential_options:type_name -> c1.connector.v2.CredentialOptions - 58, // 67: c1.connectorapi.baton.v1.Task.CreateAccountTask.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig - 52, // 68: c1.connectorapi.baton.v1.Task.CreateResourceTask.resource:type_name -> c1.connector.v2.Resource - 59, // 69: c1.connectorapi.baton.v1.Task.DeleteResourceTask.resource_id:type_name -> c1.connector.v2.ResourceId - 59, // 70: c1.connectorapi.baton.v1.Task.DeleteResourceTask.parent_resource_id:type_name -> c1.connector.v2.ResourceId - 59, // 71: c1.connectorapi.baton.v1.Task.RotateCredentialsTask.resource_id:type_name -> c1.connector.v2.ResourceId - 57, // 72: c1.connectorapi.baton.v1.Task.RotateCredentialsTask.credential_options:type_name -> c1.connector.v2.CredentialOptions - 58, // 73: c1.connectorapi.baton.v1.Task.RotateCredentialsTask.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig - 59, // 74: c1.connectorapi.baton.v1.Task.IssueCredentialTask.identity_id:type_name -> c1.connector.v2.ResourceId - 60, // 75: c1.connectorapi.baton.v1.Task.IssueCredentialTask.credential_options:type_name -> c1.connector.v2.CredentialIssueOptions - 58, // 76: c1.connectorapi.baton.v1.Task.IssueCredentialTask.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig - 53, // 77: c1.connectorapi.baton.v1.Task.IssueCredentialTask.expires_at:type_name -> google.protobuf.Timestamp - 61, // 78: c1.connectorapi.baton.v1.Task.CreateTicketTask.ticket_request:type_name -> c1.connector.v2.TicketRequest - 62, // 79: c1.connectorapi.baton.v1.Task.CreateTicketTask.ticket_schema:type_name -> c1.connector.v2.TicketSchema - 49, // 80: c1.connectorapi.baton.v1.Task.CreateTicketTask.annotations:type_name -> google.protobuf.Any - 29, // 81: c1.connectorapi.baton.v1.Task.BulkCreateTicketsTask.ticket_requests:type_name -> c1.connectorapi.baton.v1.Task.CreateTicketTask - 33, // 82: c1.connectorapi.baton.v1.Task.BulkGetTicketsTask.ticket_requests:type_name -> c1.connectorapi.baton.v1.Task.GetTicketTask - 49, // 83: c1.connectorapi.baton.v1.Task.ListTicketSchemasTask.annotations:type_name -> google.protobuf.Any - 49, // 84: c1.connectorapi.baton.v1.Task.GetTicketTask.annotations:type_name -> google.protobuf.Any - 49, // 85: c1.connectorapi.baton.v1.Task.ActionListSchemasTask.annotations:type_name -> google.protobuf.Any - 49, // 86: c1.connectorapi.baton.v1.Task.ActionGetSchemaTask.annotations:type_name -> google.protobuf.Any - 63, // 87: c1.connectorapi.baton.v1.Task.ActionInvokeTask.args:type_name -> google.protobuf.Struct - 49, // 88: c1.connectorapi.baton.v1.Task.ActionInvokeTask.annotations:type_name -> google.protobuf.Any - 49, // 89: c1.connectorapi.baton.v1.Task.ActionStatusTask.annotations:type_name -> google.protobuf.Any - 49, // 90: c1.connectorapi.baton.v1.Task.CreateSyncDiffTask.annotations:type_name -> google.protobuf.Any - 40, // 91: c1.connectorapi.baton.v1.Task.CompactSyncs.compactable_syncs:type_name -> c1.connectorapi.baton.v1.Task.CompactSyncs.CompactableSync - 49, // 92: c1.connectorapi.baton.v1.Task.CompactSyncs.annotations:type_name -> google.protobuf.Any - 49, // 93: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadMetadata.annotations:type_name -> google.protobuf.Any - 49, // 94: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadEOF.annotations:type_name -> google.protobuf.Any - 49, // 95: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.annotations:type_name -> google.protobuf.Any - 49, // 96: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.response:type_name -> google.protobuf.Any - 49, // 97: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.annotations:type_name -> google.protobuf.Any - 49, // 98: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.response:type_name -> google.protobuf.Any - 2, // 99: c1.connectorapi.baton.v1.BatonService.Hello:input_type -> c1.connectorapi.baton.v1.BatonServiceHelloRequest - 4, // 100: c1.connectorapi.baton.v1.BatonService.GetTask:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskRequest - 5, // 101: c1.connectorapi.baton.v1.BatonService.GetTasks:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksRequest - 8, // 102: c1.connectorapi.baton.v1.BatonService.Heartbeat:input_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatRequest - 12, // 103: c1.connectorapi.baton.v1.BatonService.FinishTask:input_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest - 10, // 104: c1.connectorapi.baton.v1.BatonService.UploadAsset:input_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest - 14, // 105: c1.connectorapi.baton.v1.BatonService.StartDebugging:input_type -> c1.connectorapi.baton.v1.StartDebuggingRequest - 3, // 106: c1.connectorapi.baton.v1.BatonService.Hello:output_type -> c1.connectorapi.baton.v1.BatonServiceHelloResponse - 7, // 107: c1.connectorapi.baton.v1.BatonService.GetTask:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskResponse - 6, // 108: c1.connectorapi.baton.v1.BatonService.GetTasks:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksResponse - 9, // 109: c1.connectorapi.baton.v1.BatonService.Heartbeat:output_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse - 13, // 110: c1.connectorapi.baton.v1.BatonService.FinishTask:output_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskResponse - 11, // 111: c1.connectorapi.baton.v1.BatonService.UploadAsset:output_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetResponse - 15, // 112: c1.connectorapi.baton.v1.BatonService.StartDebugging:output_type -> c1.connectorapi.baton.v1.StartDebuggingResponse - 106, // [106:113] is the sub-list for method output_type - 99, // [99:106] is the sub-list for method input_type - 99, // [99:99] is the sub-list for extension type_name - 99, // [99:99] is the sub-list for extension extendee - 0, // [0:99] is the sub-list for field type_name + 39, // 20: c1.connectorapi.baton.v1.Task.compact_syncs:type_name -> c1.connectorapi.baton.v1.Task.CompactSyncs + 21, // 21: c1.connectorapi.baton.v1.Task.list_event_feeds:type_name -> c1.connectorapi.baton.v1.Task.ListEventFeedsTask + 20, // 22: c1.connectorapi.baton.v1.Task.list_events:type_name -> c1.connectorapi.baton.v1.Task.ListEventsTask + 28, // 23: c1.connectorapi.baton.v1.Task.issue_credential:type_name -> c1.connectorapi.baton.v1.Task.IssueCredentialTask + 41, // 24: c1.connectorapi.baton.v1.BatonServiceHelloRequest.build_info:type_name -> c1.connectorapi.baton.v1.BatonServiceHelloRequest.BuildInfo + 42, // 25: c1.connectorapi.baton.v1.BatonServiceHelloRequest.os_info:type_name -> c1.connectorapi.baton.v1.BatonServiceHelloRequest.OSInfo + 48, // 26: c1.connectorapi.baton.v1.BatonServiceHelloRequest.connector_metadata:type_name -> c1.connector.v2.ConnectorMetadata + 49, // 27: c1.connectorapi.baton.v1.BatonServiceHelloRequest.annotations:type_name -> google.protobuf.Any + 49, // 28: c1.connectorapi.baton.v1.BatonServiceHelloResponse.annotations:type_name -> google.protobuf.Any + 49, // 29: c1.connectorapi.baton.v1.BatonServiceGetTasksRequest.annotations:type_name -> google.protobuf.Any + 1, // 30: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.tasks:type_name -> c1.connectorapi.baton.v1.Task + 50, // 31: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.next_poll:type_name -> google.protobuf.Duration + 50, // 32: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.next_heartbeat:type_name -> google.protobuf.Duration + 49, // 33: c1.connectorapi.baton.v1.BatonServiceGetTasksResponse.annotations:type_name -> google.protobuf.Any + 1, // 34: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.task:type_name -> c1.connectorapi.baton.v1.Task + 50, // 35: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.next_poll:type_name -> google.protobuf.Duration + 50, // 36: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.next_heartbeat:type_name -> google.protobuf.Duration + 49, // 37: c1.connectorapi.baton.v1.BatonServiceGetTaskResponse.annotations:type_name -> google.protobuf.Any + 49, // 38: c1.connectorapi.baton.v1.BatonServiceHeartbeatRequest.annotations:type_name -> google.protobuf.Any + 50, // 39: c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse.next_heartbeat:type_name -> google.protobuf.Duration + 49, // 40: c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse.annotations:type_name -> google.protobuf.Any + 43, // 41: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.metadata:type_name -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadMetadata + 44, // 42: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.data:type_name -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadData + 45, // 43: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.eof:type_name -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadEOF + 49, // 44: c1.connectorapi.baton.v1.BatonServiceUploadAssetResponse.annotations:type_name -> google.protobuf.Any + 51, // 45: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.status:type_name -> google.rpc.Status + 46, // 46: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.error:type_name -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error + 47, // 47: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.success:type_name -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success + 49, // 48: c1.connectorapi.baton.v1.BatonServiceFinishTaskResponse.annotations:type_name -> google.protobuf.Any + 49, // 49: c1.connectorapi.baton.v1.Task.NoneTask.annotations:type_name -> google.protobuf.Any + 49, // 50: c1.connectorapi.baton.v1.Task.HelloTask.annotations:type_name -> google.protobuf.Any + 49, // 51: c1.connectorapi.baton.v1.Task.SyncFullTask.annotations:type_name -> google.protobuf.Any + 52, // 52: c1.connectorapi.baton.v1.Task.SyncFullTask.targeted_sync_resources:type_name -> c1.connector.v2.Resource + 49, // 53: c1.connectorapi.baton.v1.Task.EventFeedTask.annotations:type_name -> google.protobuf.Any + 53, // 54: c1.connectorapi.baton.v1.Task.EventFeedTask.start_at:type_name -> google.protobuf.Timestamp + 49, // 55: c1.connectorapi.baton.v1.Task.ListEventsTask.annotations:type_name -> google.protobuf.Any + 53, // 56: c1.connectorapi.baton.v1.Task.ListEventsTask.start_at:type_name -> google.protobuf.Timestamp + 49, // 57: c1.connectorapi.baton.v1.Task.ListEventFeedsTask.annotations:type_name -> google.protobuf.Any + 54, // 58: c1.connectorapi.baton.v1.Task.GrantTask.entitlement:type_name -> c1.connector.v2.Entitlement + 52, // 59: c1.connectorapi.baton.v1.Task.GrantTask.principal:type_name -> c1.connector.v2.Resource + 49, // 60: c1.connectorapi.baton.v1.Task.GrantTask.annotations:type_name -> google.protobuf.Any + 50, // 61: c1.connectorapi.baton.v1.Task.GrantTask.duration:type_name -> google.protobuf.Duration + 55, // 62: c1.connectorapi.baton.v1.Task.RevokeTask.grant:type_name -> c1.connector.v2.Grant + 49, // 63: c1.connectorapi.baton.v1.Task.RevokeTask.annotations:type_name -> google.protobuf.Any + 56, // 64: c1.connectorapi.baton.v1.Task.CreateAccountTask.account_info:type_name -> c1.connector.v2.AccountInfo + 57, // 65: c1.connectorapi.baton.v1.Task.CreateAccountTask.credential_options:type_name -> c1.connector.v2.CredentialOptions + 58, // 66: c1.connectorapi.baton.v1.Task.CreateAccountTask.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig + 52, // 67: c1.connectorapi.baton.v1.Task.CreateResourceTask.resource:type_name -> c1.connector.v2.Resource + 59, // 68: c1.connectorapi.baton.v1.Task.DeleteResourceTask.resource_id:type_name -> c1.connector.v2.ResourceId + 59, // 69: c1.connectorapi.baton.v1.Task.DeleteResourceTask.parent_resource_id:type_name -> c1.connector.v2.ResourceId + 59, // 70: c1.connectorapi.baton.v1.Task.RotateCredentialsTask.resource_id:type_name -> c1.connector.v2.ResourceId + 57, // 71: c1.connectorapi.baton.v1.Task.RotateCredentialsTask.credential_options:type_name -> c1.connector.v2.CredentialOptions + 58, // 72: c1.connectorapi.baton.v1.Task.RotateCredentialsTask.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig + 59, // 73: c1.connectorapi.baton.v1.Task.IssueCredentialTask.identity_id:type_name -> c1.connector.v2.ResourceId + 60, // 74: c1.connectorapi.baton.v1.Task.IssueCredentialTask.credential_options:type_name -> c1.connector.v2.CredentialIssueOptions + 58, // 75: c1.connectorapi.baton.v1.Task.IssueCredentialTask.encryption_configs:type_name -> c1.connector.v2.EncryptionConfig + 53, // 76: c1.connectorapi.baton.v1.Task.IssueCredentialTask.expires_at:type_name -> google.protobuf.Timestamp + 61, // 77: c1.connectorapi.baton.v1.Task.CreateTicketTask.ticket_request:type_name -> c1.connector.v2.TicketRequest + 62, // 78: c1.connectorapi.baton.v1.Task.CreateTicketTask.ticket_schema:type_name -> c1.connector.v2.TicketSchema + 49, // 79: c1.connectorapi.baton.v1.Task.CreateTicketTask.annotations:type_name -> google.protobuf.Any + 29, // 80: c1.connectorapi.baton.v1.Task.BulkCreateTicketsTask.ticket_requests:type_name -> c1.connectorapi.baton.v1.Task.CreateTicketTask + 33, // 81: c1.connectorapi.baton.v1.Task.BulkGetTicketsTask.ticket_requests:type_name -> c1.connectorapi.baton.v1.Task.GetTicketTask + 49, // 82: c1.connectorapi.baton.v1.Task.ListTicketSchemasTask.annotations:type_name -> google.protobuf.Any + 49, // 83: c1.connectorapi.baton.v1.Task.GetTicketTask.annotations:type_name -> google.protobuf.Any + 49, // 84: c1.connectorapi.baton.v1.Task.ActionListSchemasTask.annotations:type_name -> google.protobuf.Any + 49, // 85: c1.connectorapi.baton.v1.Task.ActionGetSchemaTask.annotations:type_name -> google.protobuf.Any + 63, // 86: c1.connectorapi.baton.v1.Task.ActionInvokeTask.args:type_name -> google.protobuf.Struct + 49, // 87: c1.connectorapi.baton.v1.Task.ActionInvokeTask.annotations:type_name -> google.protobuf.Any + 49, // 88: c1.connectorapi.baton.v1.Task.ActionStatusTask.annotations:type_name -> google.protobuf.Any + 49, // 89: c1.connectorapi.baton.v1.Task.CreateSyncDiffTask.annotations:type_name -> google.protobuf.Any + 40, // 90: c1.connectorapi.baton.v1.Task.CompactSyncs.compactable_syncs:type_name -> c1.connectorapi.baton.v1.Task.CompactSyncs.CompactableSync + 49, // 91: c1.connectorapi.baton.v1.Task.CompactSyncs.annotations:type_name -> google.protobuf.Any + 49, // 92: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadMetadata.annotations:type_name -> google.protobuf.Any + 49, // 93: c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest.UploadEOF.annotations:type_name -> google.protobuf.Any + 49, // 94: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.annotations:type_name -> google.protobuf.Any + 49, // 95: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Error.response:type_name -> google.protobuf.Any + 49, // 96: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.annotations:type_name -> google.protobuf.Any + 49, // 97: c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest.Success.response:type_name -> google.protobuf.Any + 2, // 98: c1.connectorapi.baton.v1.BatonService.Hello:input_type -> c1.connectorapi.baton.v1.BatonServiceHelloRequest + 4, // 99: c1.connectorapi.baton.v1.BatonService.GetTask:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskRequest + 5, // 100: c1.connectorapi.baton.v1.BatonService.GetTasks:input_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksRequest + 8, // 101: c1.connectorapi.baton.v1.BatonService.Heartbeat:input_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatRequest + 12, // 102: c1.connectorapi.baton.v1.BatonService.FinishTask:input_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskRequest + 10, // 103: c1.connectorapi.baton.v1.BatonService.UploadAsset:input_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetRequest + 14, // 104: c1.connectorapi.baton.v1.BatonService.StartDebugging:input_type -> c1.connectorapi.baton.v1.StartDebuggingRequest + 3, // 105: c1.connectorapi.baton.v1.BatonService.Hello:output_type -> c1.connectorapi.baton.v1.BatonServiceHelloResponse + 7, // 106: c1.connectorapi.baton.v1.BatonService.GetTask:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTaskResponse + 6, // 107: c1.connectorapi.baton.v1.BatonService.GetTasks:output_type -> c1.connectorapi.baton.v1.BatonServiceGetTasksResponse + 9, // 108: c1.connectorapi.baton.v1.BatonService.Heartbeat:output_type -> c1.connectorapi.baton.v1.BatonServiceHeartbeatResponse + 13, // 109: c1.connectorapi.baton.v1.BatonService.FinishTask:output_type -> c1.connectorapi.baton.v1.BatonServiceFinishTaskResponse + 11, // 110: c1.connectorapi.baton.v1.BatonService.UploadAsset:output_type -> c1.connectorapi.baton.v1.BatonServiceUploadAssetResponse + 15, // 111: c1.connectorapi.baton.v1.BatonService.StartDebugging:output_type -> c1.connectorapi.baton.v1.StartDebuggingResponse + 105, // [105:112] is the sub-list for method output_type + 98, // [98:105] is the sub-list for method input_type + 98, // [98:98] is the sub-list for extension type_name + 98, // [98:98] is the sub-list for extension extendee + 0, // [0:98] is the sub-list for field type_name } func init() { file_c1_connectorapi_baton_v1_baton_proto_init() } @@ -6100,7 +6059,6 @@ func file_c1_connectorapi_baton_v1_baton_proto_init() { (*task_ActionGetSchema)(nil), (*task_ActionInvoke)(nil), (*task_ActionStatus)(nil), - (*task_CreateSyncDiff)(nil), (*task_CompactSyncs_)(nil), (*task_ListEventFeeds)(nil), (*task_ListEvents)(nil), diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go index 13247122..5eb228fb 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go @@ -40,11 +40,18 @@ const ( type SyncType int32 const ( - SyncType_SYNC_TYPE_UNSPECIFIED SyncType = 0 - SyncType_SYNC_TYPE_FULL SyncType = 1 - SyncType_SYNC_TYPE_PARTIAL SyncType = 2 - SyncType_SYNC_TYPE_RESOURCES_ONLY SyncType = 3 - SyncType_SYNC_TYPE_PARTIAL_UPSERTS SyncType = 4 + SyncType_SYNC_TYPE_UNSPECIFIED SyncType = 0 + SyncType_SYNC_TYPE_FULL SyncType = 1 + SyncType_SYNC_TYPE_PARTIAL SyncType = 2 + SyncType_SYNC_TYPE_RESOURCES_ONLY SyncType = 3 + // Deprecated: 4 and 5 were the diff-sync pair; diff-sync support was + // removed and nothing produces or consumes them. Kept (rather than + // reserved) so the buf breaking policy can keep forbidding enum value + // deletion repo-wide. + // + // Deprecated: Marked as deprecated in c1/storage/v3/records.proto. + SyncType_SYNC_TYPE_PARTIAL_UPSERTS SyncType = 4 + // Deprecated: Marked as deprecated in c1/storage/v3/records.proto. SyncType_SYNC_TYPE_PARTIAL_DELETIONS SyncType = 5 ) @@ -591,8 +598,10 @@ type ResourceRecord struct { // to 12 when profile/status/created_at (9-11) landed on main first. // No artifact was ever written with the old number. SourceScopeKey string `protobuf:"bytes,12,opt,name=source_scope_key,json=sourceScopeKey,proto3" json:"source_scope_key,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // External ID of the resource icon asset. This must point to an asset that is an image. + IconAssetExternalId string `protobuf:"bytes,13,opt,name=icon_asset_external_id,json=iconAssetExternalId,proto3" json:"icon_asset_external_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ResourceRecord) Reset() { @@ -697,6 +706,13 @@ func (x *ResourceRecord) GetSourceScopeKey() string { return "" } +func (x *ResourceRecord) GetIconAssetExternalId() string { + if x != nil { + return x.IconAssetExternalId + } + return "" +} + func (x *ResourceRecord) SetResourceTypeId(v string) { x.ResourceTypeId = v } @@ -741,6 +757,10 @@ func (x *ResourceRecord) SetSourceScopeKey(v string) { x.SourceScopeKey = v } +func (x *ResourceRecord) SetIconAssetExternalId(v string) { + x.IconAssetExternalId = v +} + func (x *ResourceRecord) HasParent() bool { if x == nil { return false @@ -818,6 +838,8 @@ type ResourceRecord_builder struct { // to 12 when profile/status/created_at (9-11) landed on main first. // No artifact was ever written with the old number. SourceScopeKey string + // External ID of the resource icon asset. This must point to an asset that is an image. + IconAssetExternalId string } func (b0 ResourceRecord_builder) Build() *ResourceRecord { @@ -835,6 +857,7 @@ func (b0 ResourceRecord_builder) Build() *ResourceRecord { x.Status = b.Status x.CreatedAt = b.CreatedAt x.SourceScopeKey = b.SourceScopeKey + x.IconAssetExternalId = b.IconAssetExternalId return m0 } @@ -1455,8 +1478,11 @@ type SyncRunRecord struct { StartedAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` EndedAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=ended_at,json=endedAt,proto3" json:"ended_at,omitempty"` SyncToken string `protobuf:"bytes,6,opt,name=sync_token,json=syncToken,proto3" json:"sync_token,omitempty"` - SupportsDiff bool `protobuf:"varint,7,opt,name=supports_diff,json=supportsDiff,proto3" json:"supports_diff,omitempty"` - LinkedSyncId string `protobuf:"bytes,8,opt,name=linked_sync_id,json=linkedSyncId,proto3" json:"linked_sync_id,omitempty"` + // supports_diff marks a sync whose data collection completed with + // SQL-layer grant metadata populated. The name is historical (it once + // gated diff-sync generation, since removed); today it gates + // `baton rollback-expansion`. + SupportsDiff bool `protobuf:"varint,7,opt,name=supports_diff,json=supportsDiff,proto3" json:"supports_diff,omitempty"` // compacted marks a sync produced by compaction (fold or rebuild) // rather than by a real connector run. Compacted artifacts are // keep-newer UPSERT merges — base rows a newer input deleted survive — @@ -1561,13 +1587,6 @@ func (x *SyncRunRecord) GetSupportsDiff() bool { return false } -func (x *SyncRunRecord) GetLinkedSyncId() string { - if x != nil { - return x.LinkedSyncId - } - return "" -} - func (x *SyncRunRecord) GetCompacted() bool { if x != nil { return x.Compacted @@ -1624,10 +1643,6 @@ func (x *SyncRunRecord) SetSupportsDiff(v bool) { x.SupportsDiff = v } -func (x *SyncRunRecord) SetLinkedSyncId(v string) { - x.LinkedSyncId = v -} - func (x *SyncRunRecord) SetCompacted(v bool) { x.Compacted = v } @@ -1675,8 +1690,11 @@ type SyncRunRecord_builder struct { StartedAt *timestamppb.Timestamp EndedAt *timestamppb.Timestamp SyncToken string + // supports_diff marks a sync whose data collection completed with + // SQL-layer grant metadata populated. The name is historical (it once + // gated diff-sync generation, since removed); today it gates + // `baton rollback-expansion`. SupportsDiff bool - LinkedSyncId string // compacted marks a sync produced by compaction (fold or rebuild) // rather than by a real connector run. Compacted artifacts are // keep-newer UPSERT merges — base rows a newer input deleted survive — @@ -1716,7 +1734,6 @@ func (b0 SyncRunRecord_builder) Build() *SyncRunRecord { x.EndedAt = b.EndedAt x.SyncToken = b.SyncToken x.SupportsDiff = b.SupportsDiff - x.LinkedSyncId = b.LinkedSyncId x.Compacted = b.Compacted x.IngestInvariantGeneration = b.IngestInvariantGeneration x.IngestInvariantCoverage = b.IngestInvariantCoverage @@ -2448,7 +2465,18 @@ type SourceCacheEntryRecord struct { // invalidated entry as a miss (the scope re-fetches cold and converges // with a cold sync); the entry itself is kept so the scope's surviving // stamped rows do not read as an I6 orphan (lost manifest write). - Invalidated bool `protobuf:"varint,5,opt,name=invalidated,proto3" json:"invalidated,omitempty"` + Invalidated bool `protobuf:"varint,5,opt,name=invalidated,proto3" json:"invalidated,omitempty"` + // Number of primary rows stamped with this scope at seal time, + // recomputed by EndSync from the primary keyspace (never maintained + // incrementally). Replay preflight requires the scope's index + // cardinality to equal this count before mutating the destination; + // a replay-eligible entry WITHOUT a count is a hard preflight error + // (seal-invariant violation — CO-004 shipped with the manifest format, + // so no counting-free artifact population exists). Presence is + // explicit so zero remains distinguishable from absent: zero means a + // proven empty scope. Cleared when a completed sync is rebound for + // mutation and recomputed when it reseals. + RowCount *uint64 `protobuf:"varint,6,opt,name=row_count,json=rowCount,proto3,oneof" json:"row_count,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2513,6 +2541,13 @@ func (x *SourceCacheEntryRecord) GetInvalidated() bool { return false } +func (x *SourceCacheEntryRecord) GetRowCount() uint64 { + if x != nil && x.RowCount != nil { + return *x.RowCount + } + return 0 +} + func (x *SourceCacheEntryRecord) SetRowKind(v string) { x.RowKind = v } @@ -2533,6 +2568,10 @@ func (x *SourceCacheEntryRecord) SetInvalidated(v bool) { x.Invalidated = v } +func (x *SourceCacheEntryRecord) SetRowCount(v uint64) { + x.RowCount = &v +} + func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { if x == nil { return false @@ -2540,10 +2579,21 @@ func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { return x.DiscoveredAt != nil } +func (x *SourceCacheEntryRecord) HasRowCount() bool { + if x == nil { + return false + } + return x.RowCount != nil +} + func (x *SourceCacheEntryRecord) ClearDiscoveredAt() { x.DiscoveredAt = nil } +func (x *SourceCacheEntryRecord) ClearRowCount() { + x.RowCount = nil +} + type SourceCacheEntryRecord_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. @@ -2564,6 +2614,17 @@ type SourceCacheEntryRecord_builder struct { // with a cold sync); the entry itself is kept so the scope's surviving // stamped rows do not read as an I6 orphan (lost manifest write). Invalidated bool + // Number of primary rows stamped with this scope at seal time, + // recomputed by EndSync from the primary keyspace (never maintained + // incrementally). Replay preflight requires the scope's index + // cardinality to equal this count before mutating the destination; + // a replay-eligible entry WITHOUT a count is a hard preflight error + // (seal-invariant violation — CO-004 shipped with the manifest format, + // so no counting-free artifact population exists). Presence is + // explicit so zero remains distinguishable from absent: zero means a + // proven empty scope. Cleared when a completed sync is rebound for + // mutation and recomputed when it reseals. + RowCount *uint64 } func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { @@ -2575,6 +2636,7 @@ func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { x.CacheValidator = b.CacheValidator x.DiscoveredAt = b.DiscoveredAt x.Invalidated = b.Invalidated + x.RowCount = b.RowCount return m0 } @@ -2756,7 +2818,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + "\vdescription\x18\a \x01(\tR\vdescription\x12-\n" + "\x12sourced_externally\x18\b \x01(\bR\x11sourcedExternally:!\x82\xf9+\x1d\n" + - "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xb8\x05\n" + + "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xed\x05\n" + "\x0eResourceRecord\x12(\n" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x12\x1f\n" + "\vresource_id\x18\x03 \x01(\tR\n" + @@ -2773,7 +2835,8 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\n" + "created_at\x18\v \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x12W\n" + "\x10source_scope_key\x18\f \x01(\tB-\x8a\xf9+)\n" + - "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey:.\x82\xf9+*\n" + + "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey\x123\n" + + "\x16icon_asset_external_id\x18\r \x01(\tR\x13iconAssetExternalId:.\x82\xf9+*\n" + "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xd7\x04\n" + "\x11EntitlementRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + @@ -2818,7 +2881,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\fcontent_type\x18\x03 \x01(\tR\vcontentType\x12\x12\n" + "\x04data\x18\x04 \x01(\fR\x04data\x12?\n" + "\rdiscovered_at\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:\"\x82\xf9+\x1e\n" + - "\x06assets\x12\async_id\x12\vexternal_id\"\xbf\x04\n" + + "\x06assets\x12\async_id\x12\vexternal_id\"\xaf\x04\n" + "\rSyncRunRecord\x12\x17\n" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12+\n" + "\x04type\x18\x02 \x01(\x0e2\x17.c1.storage.v3.SyncTypeR\x04type\x12$\n" + @@ -2828,14 +2891,13 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\bended_at\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\aendedAt\x12\x1d\n" + "\n" + "sync_token\x18\x06 \x01(\tR\tsyncToken\x12#\n" + - "\rsupports_diff\x18\a \x01(\bR\fsupportsDiff\x12$\n" + - "\x0elinked_sync_id\x18\b \x01(\tR\flinkedSyncId\x12\x1c\n" + + "\rsupports_diff\x18\a \x01(\bR\fsupportsDiff\x12\x1c\n" + "\tcompacted\x18\t \x01(\bR\tcompacted\x12>\n" + "\x1bingest_invariant_generation\x18\n" + " \x01(\tR\x19ingestInvariantGeneration\x12:\n" + "\x19ingest_invariant_coverage\x18\v \x03(\tR\x17ingestInvariantCoverage\x122\n" + "\x15ingest_invariant_mode\x18\f \x01(\tR\x13ingestInvariantMode:\x18\x82\xf9+\x14\n" + - "\tsync_runs\x12\async_id\"\xfe\v\n" + + "\tsync_runs\x12\async_idJ\x04\b\b\x10\tR\x0elinked_sync_id\"\xfe\v\n" + "\x0fSyncStatsRecord\x12\x17\n" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12%\n" + "\x0eresource_types\x18\x02 \x01(\x03R\rresourceTypes\x12\x1c\n" + @@ -2893,28 +2955,31 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key\"\x8d\x02\n" + + "\bsessions\x12\async_id\x12\x03key\"\xbd\x02\n" + "\x16SourceCacheEntryRecord\x12\x19\n" + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1b\n" + "\tscope_key\x18\x02 \x01(\tR\bscopeKey\x12'\n" + "\x0fcache_validator\x18\x03 \x01(\tR\x0ecacheValidator\x12?\n" + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + - "\vinvalidated\x18\x05 \x01(\bR\vinvalidated:/\x82\xf9++\n" + - "\x14source_cache_entries\x12\brow_kind\x12\tscope_key\"\xcc\x02\n" + + "\vinvalidated\x18\x05 \x01(\bR\vinvalidated\x12 \n" + + "\trow_count\x18\x06 \x01(\x04H\x00R\browCount\x88\x01\x01:/\x82\xf9++\n" + + "\x14source_cache_entries\x12\brow_kind\x12\tscope_keyB\f\n" + + "\n" + + "_row_count\"\xcc\x02\n" + "\x17SourceCacheCompatRecord\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12<\n" + "\x1aconnector_cache_generation\x18\x02 \x01(\tR\x18connectorCacheGeneration\x12@\n" + "\x1cconnector_config_fingerprint\x18\x03 \x01(\tR\x1aconnectorConfigFingerprint\x12D\n" + "\x1esdk_materialization_generation\x18\x04 \x01(\tR\x1csdkMaterializationGeneration\x12<\n" + "\x1async_selection_fingerprint\x18\x05 \x01(\tR\x18syncSelectionFingerprint:\x1d\x82\xf9+\x19\n" + - "\x13source_cache_compat\x12\x02id*\xae\x01\n" + + "\x13source_cache_compat\x12\x02id*\xb6\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + "\x11SYNC_TYPE_PARTIAL\x10\x02\x12\x1c\n" + - "\x18SYNC_TYPE_RESOURCES_ONLY\x10\x03\x12\x1d\n" + - "\x19SYNC_TYPE_PARTIAL_UPSERTS\x10\x04\x12\x1f\n" + - "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" + "\x18SYNC_TYPE_RESOURCES_ONLY\x10\x03\x12!\n" + + "\x19SYNC_TYPE_PARTIAL_UPSERTS\x10\x04\x1a\x02\b\x01\x12#\n" + + "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05\x1a\x02\b\x01B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" var file_c1_storage_v3_records_proto_enumTypes = make([]protoimpl.EnumInfo, 2) var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 22) @@ -2999,6 +3064,7 @@ func file_c1_storage_v3_records_proto_init() { } file_c1_storage_v3_options_proto_init() file_c1_storage_v3_refs_proto_init() + file_c1_storage_v3_records_proto_msgTypes[13].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go index e3d7f2f7..3b1948f2 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.validate.go @@ -736,6 +736,8 @@ func (m *ResourceRecord) validate(all bool) error { // no validation rules for SourceScopeKey + // no validation rules for IconAssetExternalId + if len(errors) > 0 { return ResourceRecordMultiError(errors) } @@ -1547,8 +1549,6 @@ func (m *SyncRunRecord) validate(all bool) error { // no validation rules for SupportsDiff - // no validation rules for LinkedSyncId - // no validation rules for Compacted // no validation rules for IngestInvariantGeneration @@ -2299,6 +2299,10 @@ func (m *SourceCacheEntryRecord) validate(all bool) error { // no validation rules for Invalidated + if m.RowCount != nil { + // no validation rules for RowCount + } + if len(errors) > 0 { return SourceCacheEntryRecordMultiError(errors) } diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go index e6df94ce..b276786f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go @@ -40,11 +40,18 @@ const ( type SyncType int32 const ( - SyncType_SYNC_TYPE_UNSPECIFIED SyncType = 0 - SyncType_SYNC_TYPE_FULL SyncType = 1 - SyncType_SYNC_TYPE_PARTIAL SyncType = 2 - SyncType_SYNC_TYPE_RESOURCES_ONLY SyncType = 3 - SyncType_SYNC_TYPE_PARTIAL_UPSERTS SyncType = 4 + SyncType_SYNC_TYPE_UNSPECIFIED SyncType = 0 + SyncType_SYNC_TYPE_FULL SyncType = 1 + SyncType_SYNC_TYPE_PARTIAL SyncType = 2 + SyncType_SYNC_TYPE_RESOURCES_ONLY SyncType = 3 + // Deprecated: 4 and 5 were the diff-sync pair; diff-sync support was + // removed and nothing produces or consumes them. Kept (rather than + // reserved) so the buf breaking policy can keep forbidding enum value + // deletion repo-wide. + // + // Deprecated: Marked as deprecated in c1/storage/v3/records.proto. + SyncType_SYNC_TYPE_PARTIAL_UPSERTS SyncType = 4 + // Deprecated: Marked as deprecated in c1/storage/v3/records.proto. SyncType_SYNC_TYPE_PARTIAL_DELETIONS SyncType = 5 ) @@ -564,20 +571,21 @@ func (b0 ResourceTypeRecord_builder) Build() *ResourceTypeRecord { } type ResourceRecord struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_ResourceTypeId string `protobuf:"bytes,2,opt,name=resource_type_id,json=resourceTypeId,proto3"` - xxx_hidden_ResourceId string `protobuf:"bytes,3,opt,name=resource_id,json=resourceId,proto3"` - xxx_hidden_DisplayName string `protobuf:"bytes,4,opt,name=display_name,json=displayName,proto3"` - xxx_hidden_Description string `protobuf:"bytes,5,opt,name=description,proto3"` - xxx_hidden_Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3"` - xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3"` - xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` - xxx_hidden_Profile *structpb.Struct `protobuf:"bytes,9,opt,name=profile,proto3"` - xxx_hidden_Status *StatusRecord `protobuf:"bytes,10,opt,name=status,proto3"` - xxx_hidden_CreatedAt *timestamppb.Timestamp `protobuf:"bytes,11,opt,name=created_at,json=createdAt,proto3"` - xxx_hidden_SourceScopeKey string `protobuf:"bytes,12,opt,name=source_scope_key,json=sourceScopeKey,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ResourceTypeId string `protobuf:"bytes,2,opt,name=resource_type_id,json=resourceTypeId,proto3"` + xxx_hidden_ResourceId string `protobuf:"bytes,3,opt,name=resource_id,json=resourceId,proto3"` + xxx_hidden_DisplayName string `protobuf:"bytes,4,opt,name=display_name,json=displayName,proto3"` + xxx_hidden_Description string `protobuf:"bytes,5,opt,name=description,proto3"` + xxx_hidden_Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3"` + xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` + xxx_hidden_Profile *structpb.Struct `protobuf:"bytes,9,opt,name=profile,proto3"` + xxx_hidden_Status *StatusRecord `protobuf:"bytes,10,opt,name=status,proto3"` + xxx_hidden_CreatedAt *timestamppb.Timestamp `protobuf:"bytes,11,opt,name=created_at,json=createdAt,proto3"` + xxx_hidden_SourceScopeKey string `protobuf:"bytes,12,opt,name=source_scope_key,json=sourceScopeKey,proto3"` + xxx_hidden_IconAssetExternalId string `protobuf:"bytes,13,opt,name=icon_asset_external_id,json=iconAssetExternalId,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ResourceRecord) Reset() { @@ -684,6 +692,13 @@ func (x *ResourceRecord) GetSourceScopeKey() string { return "" } +func (x *ResourceRecord) GetIconAssetExternalId() string { + if x != nil { + return x.xxx_hidden_IconAssetExternalId + } + return "" +} + func (x *ResourceRecord) SetResourceTypeId(v string) { x.xxx_hidden_ResourceTypeId = v } @@ -728,6 +743,10 @@ func (x *ResourceRecord) SetSourceScopeKey(v string) { x.xxx_hidden_SourceScopeKey = v } +func (x *ResourceRecord) SetIconAssetExternalId(v string) { + x.xxx_hidden_IconAssetExternalId = v +} + func (x *ResourceRecord) HasParent() bool { if x == nil { return false @@ -805,6 +824,8 @@ type ResourceRecord_builder struct { // to 12 when profile/status/created_at (9-11) landed on main first. // No artifact was ever written with the old number. SourceScopeKey string + // External ID of the resource icon asset. This must point to an asset that is an image. + IconAssetExternalId string } func (b0 ResourceRecord_builder) Build() *ResourceRecord { @@ -822,6 +843,7 @@ func (b0 ResourceRecord_builder) Build() *ResourceRecord { x.xxx_hidden_Status = b.Status x.xxx_hidden_CreatedAt = b.CreatedAt x.xxx_hidden_SourceScopeKey = b.SourceScopeKey + x.xxx_hidden_IconAssetExternalId = b.IconAssetExternalId return m0 } @@ -1415,7 +1437,6 @@ type SyncRunRecord struct { xxx_hidden_EndedAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=ended_at,json=endedAt,proto3"` xxx_hidden_SyncToken string `protobuf:"bytes,6,opt,name=sync_token,json=syncToken,proto3"` xxx_hidden_SupportsDiff bool `protobuf:"varint,7,opt,name=supports_diff,json=supportsDiff,proto3"` - xxx_hidden_LinkedSyncId string `protobuf:"bytes,8,opt,name=linked_sync_id,json=linkedSyncId,proto3"` xxx_hidden_Compacted bool `protobuf:"varint,9,opt,name=compacted,proto3"` xxx_hidden_IngestInvariantGeneration string `protobuf:"bytes,10,opt,name=ingest_invariant_generation,json=ingestInvariantGeneration,proto3"` xxx_hidden_IngestInvariantCoverage []string `protobuf:"bytes,11,rep,name=ingest_invariant_coverage,json=ingestInvariantCoverage,proto3"` @@ -1498,13 +1519,6 @@ func (x *SyncRunRecord) GetSupportsDiff() bool { return false } -func (x *SyncRunRecord) GetLinkedSyncId() string { - if x != nil { - return x.xxx_hidden_LinkedSyncId - } - return "" -} - func (x *SyncRunRecord) GetCompacted() bool { if x != nil { return x.xxx_hidden_Compacted @@ -1561,10 +1575,6 @@ func (x *SyncRunRecord) SetSupportsDiff(v bool) { x.xxx_hidden_SupportsDiff = v } -func (x *SyncRunRecord) SetLinkedSyncId(v string) { - x.xxx_hidden_LinkedSyncId = v -} - func (x *SyncRunRecord) SetCompacted(v bool) { x.xxx_hidden_Compacted = v } @@ -1612,8 +1622,11 @@ type SyncRunRecord_builder struct { StartedAt *timestamppb.Timestamp EndedAt *timestamppb.Timestamp SyncToken string + // supports_diff marks a sync whose data collection completed with + // SQL-layer grant metadata populated. The name is historical (it once + // gated diff-sync generation, since removed); today it gates + // `baton rollback-expansion`. SupportsDiff bool - LinkedSyncId string // compacted marks a sync produced by compaction (fold or rebuild) // rather than by a real connector run. Compacted artifacts are // keep-newer UPSERT merges — base rows a newer input deleted survive — @@ -1653,7 +1666,6 @@ func (b0 SyncRunRecord_builder) Build() *SyncRunRecord { x.xxx_hidden_EndedAt = b.EndedAt x.xxx_hidden_SyncToken = b.SyncToken x.xxx_hidden_SupportsDiff = b.SupportsDiff - x.xxx_hidden_LinkedSyncId = b.LinkedSyncId x.xxx_hidden_Compacted = b.Compacted x.xxx_hidden_IngestInvariantGeneration = b.IngestInvariantGeneration x.xxx_hidden_IngestInvariantCoverage = b.IngestInvariantCoverage @@ -2350,6 +2362,9 @@ type SourceCacheEntryRecord struct { xxx_hidden_CacheValidator string `protobuf:"bytes,3,opt,name=cache_validator,json=cacheValidator,proto3"` xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3"` xxx_hidden_Invalidated bool `protobuf:"varint,5,opt,name=invalidated,proto3"` + xxx_hidden_RowCount uint64 `protobuf:"varint,6,opt,name=row_count,json=rowCount,proto3,oneof"` + XXX_raceDetectHookData protoimpl.RaceDetectHookData + XXX_presence [1]uint32 unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -2414,6 +2429,13 @@ func (x *SourceCacheEntryRecord) GetInvalidated() bool { return false } +func (x *SourceCacheEntryRecord) GetRowCount() uint64 { + if x != nil { + return x.xxx_hidden_RowCount + } + return 0 +} + func (x *SourceCacheEntryRecord) SetRowKind(v string) { x.xxx_hidden_RowKind = v } @@ -2434,6 +2456,11 @@ func (x *SourceCacheEntryRecord) SetInvalidated(v bool) { x.xxx_hidden_Invalidated = v } +func (x *SourceCacheEntryRecord) SetRowCount(v uint64) { + x.xxx_hidden_RowCount = v + protoimpl.X.SetPresent(&(x.XXX_presence[0]), 5, 6) +} + func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { if x == nil { return false @@ -2441,10 +2468,22 @@ func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { return x.xxx_hidden_DiscoveredAt != nil } +func (x *SourceCacheEntryRecord) HasRowCount() bool { + if x == nil { + return false + } + return protoimpl.X.Present(&(x.XXX_presence[0]), 5) +} + func (x *SourceCacheEntryRecord) ClearDiscoveredAt() { x.xxx_hidden_DiscoveredAt = nil } +func (x *SourceCacheEntryRecord) ClearRowCount() { + protoimpl.X.ClearPresent(&(x.XXX_presence[0]), 5) + x.xxx_hidden_RowCount = 0 +} + type SourceCacheEntryRecord_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. @@ -2465,6 +2504,17 @@ type SourceCacheEntryRecord_builder struct { // with a cold sync); the entry itself is kept so the scope's surviving // stamped rows do not read as an I6 orphan (lost manifest write). Invalidated bool + // Number of primary rows stamped with this scope at seal time, + // recomputed by EndSync from the primary keyspace (never maintained + // incrementally). Replay preflight requires the scope's index + // cardinality to equal this count before mutating the destination; + // a replay-eligible entry WITHOUT a count is a hard preflight error + // (seal-invariant violation — CO-004 shipped with the manifest format, + // so no counting-free artifact population exists). Presence is + // explicit so zero remains distinguishable from absent: zero means a + // proven empty scope. Cleared when a completed sync is rebound for + // mutation and recomputed when it reseals. + RowCount *uint64 } func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { @@ -2476,6 +2526,10 @@ func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { x.xxx_hidden_CacheValidator = b.CacheValidator x.xxx_hidden_DiscoveredAt = b.DiscoveredAt x.xxx_hidden_Invalidated = b.Invalidated + if b.RowCount != nil { + protoimpl.X.SetPresentNonAtomic(&(x.XXX_presence[0]), 5, 6) + x.xxx_hidden_RowCount = *b.RowCount + } return m0 } @@ -2646,7 +2700,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + "\vdescription\x18\a \x01(\tR\vdescription\x12-\n" + "\x12sourced_externally\x18\b \x01(\bR\x11sourcedExternally:!\x82\xf9+\x1d\n" + - "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xb8\x05\n" + + "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xed\x05\n" + "\x0eResourceRecord\x12(\n" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x12\x1f\n" + "\vresource_id\x18\x03 \x01(\tR\n" + @@ -2663,7 +2717,8 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\n" + "created_at\x18\v \x01(\v2\x1a.google.protobuf.TimestampR\tcreatedAt\x12W\n" + "\x10source_scope_key\x18\f \x01(\tB-\x8a\xf9+)\n" + - "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey:.\x82\xf9+*\n" + + "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey\x123\n" + + "\x16icon_asset_external_id\x18\r \x01(\tR\x13iconAssetExternalId:.\x82\xf9+*\n" + "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xd7\x04\n" + "\x11EntitlementRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + @@ -2708,7 +2763,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\fcontent_type\x18\x03 \x01(\tR\vcontentType\x12\x12\n" + "\x04data\x18\x04 \x01(\fR\x04data\x12?\n" + "\rdiscovered_at\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:\"\x82\xf9+\x1e\n" + - "\x06assets\x12\async_id\x12\vexternal_id\"\xbf\x04\n" + + "\x06assets\x12\async_id\x12\vexternal_id\"\xaf\x04\n" + "\rSyncRunRecord\x12\x17\n" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12+\n" + "\x04type\x18\x02 \x01(\x0e2\x17.c1.storage.v3.SyncTypeR\x04type\x12$\n" + @@ -2718,14 +2773,13 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\bended_at\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\aendedAt\x12\x1d\n" + "\n" + "sync_token\x18\x06 \x01(\tR\tsyncToken\x12#\n" + - "\rsupports_diff\x18\a \x01(\bR\fsupportsDiff\x12$\n" + - "\x0elinked_sync_id\x18\b \x01(\tR\flinkedSyncId\x12\x1c\n" + + "\rsupports_diff\x18\a \x01(\bR\fsupportsDiff\x12\x1c\n" + "\tcompacted\x18\t \x01(\bR\tcompacted\x12>\n" + "\x1bingest_invariant_generation\x18\n" + " \x01(\tR\x19ingestInvariantGeneration\x12:\n" + "\x19ingest_invariant_coverage\x18\v \x03(\tR\x17ingestInvariantCoverage\x122\n" + "\x15ingest_invariant_mode\x18\f \x01(\tR\x13ingestInvariantMode:\x18\x82\xf9+\x14\n" + - "\tsync_runs\x12\async_id\"\xfe\v\n" + + "\tsync_runs\x12\async_idJ\x04\b\b\x10\tR\x0elinked_sync_id\"\xfe\v\n" + "\x0fSyncStatsRecord\x12\x17\n" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12%\n" + "\x0eresource_types\x18\x02 \x01(\x03R\rresourceTypes\x12\x1c\n" + @@ -2783,28 +2837,31 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key\"\x8d\x02\n" + + "\bsessions\x12\async_id\x12\x03key\"\xbd\x02\n" + "\x16SourceCacheEntryRecord\x12\x19\n" + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1b\n" + "\tscope_key\x18\x02 \x01(\tR\bscopeKey\x12'\n" + "\x0fcache_validator\x18\x03 \x01(\tR\x0ecacheValidator\x12?\n" + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + - "\vinvalidated\x18\x05 \x01(\bR\vinvalidated:/\x82\xf9++\n" + - "\x14source_cache_entries\x12\brow_kind\x12\tscope_key\"\xcc\x02\n" + + "\vinvalidated\x18\x05 \x01(\bR\vinvalidated\x12 \n" + + "\trow_count\x18\x06 \x01(\x04H\x00R\browCount\x88\x01\x01:/\x82\xf9++\n" + + "\x14source_cache_entries\x12\brow_kind\x12\tscope_keyB\f\n" + + "\n" + + "_row_count\"\xcc\x02\n" + "\x17SourceCacheCompatRecord\x12\x0e\n" + "\x02id\x18\x01 \x01(\tR\x02id\x12<\n" + "\x1aconnector_cache_generation\x18\x02 \x01(\tR\x18connectorCacheGeneration\x12@\n" + "\x1cconnector_config_fingerprint\x18\x03 \x01(\tR\x1aconnectorConfigFingerprint\x12D\n" + "\x1esdk_materialization_generation\x18\x04 \x01(\tR\x1csdkMaterializationGeneration\x12<\n" + "\x1async_selection_fingerprint\x18\x05 \x01(\tR\x18syncSelectionFingerprint:\x1d\x82\xf9+\x19\n" + - "\x13source_cache_compat\x12\x02id*\xae\x01\n" + + "\x13source_cache_compat\x12\x02id*\xb6\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + "\x11SYNC_TYPE_PARTIAL\x10\x02\x12\x1c\n" + - "\x18SYNC_TYPE_RESOURCES_ONLY\x10\x03\x12\x1d\n" + - "\x19SYNC_TYPE_PARTIAL_UPSERTS\x10\x04\x12\x1f\n" + - "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" + "\x18SYNC_TYPE_RESOURCES_ONLY\x10\x03\x12!\n" + + "\x19SYNC_TYPE_PARTIAL_UPSERTS\x10\x04\x1a\x02\b\x01\x12#\n" + + "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05\x1a\x02\b\x01B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" var file_c1_storage_v3_records_proto_enumTypes = make([]protoimpl.EnumInfo, 2) var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 22) @@ -2889,6 +2946,7 @@ func file_c1_storage_v3_records_proto_init() { } file_c1_storage_v3_options_proto_init() file_c1_storage_v3_refs_proto_init() + file_c1_storage_v3_records_proto_msgTypes[13].OneofWrappers = []any{} type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go b/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go index 22386800..2455ba43 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/cli/commands.go @@ -349,14 +349,6 @@ func MakeMainCommand[T field.Configurable]( opts = append(opts, connectorrunner.WithTicketingEnabled(), connectorrunner.WithGetTicket(v.GetString("ticket-id"))) - case v.GetBool("diff-syncs"): - opts = append(opts, - connectorrunner.WithDiffSyncs( - v.GetString("file"), - v.GetString("base-sync-id"), - v.GetString("applied-sync-id"), - ), - ) case v.GetBool("compact-syncs"): opts = append(opts, connectorrunner.WithSyncCompactor( diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go b/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go index f8c7c77b..200392a0 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/config/config.go @@ -12,6 +12,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/cli" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" "github.com/conductorone/baton-sdk/pkg/connectorrunner" + "github.com/conductorone/baton-sdk/pkg/exit" "github.com/conductorone/baton-sdk/pkg/field" "github.com/conductorone/baton-sdk/pkg/types" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" @@ -60,8 +61,7 @@ func RunConnector[T field.Configurable]( _, cmd, err := DefineConfigurationV2(ctx, connectorName, f, schema, options...) if err != nil { - fmt.Fprintln(os.Stderr, err.Error()) - os.Exit(1) + exit.LogExit(err) return } @@ -69,8 +69,7 @@ func RunConnector[T field.Configurable]( err = cmd.Execute() if err != nil { - fmt.Fprintln(os.Stderr, err.Error()) - os.Exit(1) + exit.LogExit(err) } } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go index 4610386f..f0fd9358 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/connectorbuilder.go @@ -530,7 +530,9 @@ func validateCredentialIssueCapabilityDetails(issue *v2.CredentialDetailsCredent if issue == nil || issue.GetPreferredOption() == v2.CapabilityDetailCredentialOption_CAPABILITY_DETAIL_CREDENTIAL_OPTION_UNSPECIFIED { return status.Error(codes.InvalidArgument, "preferred credential issue option is not set") } - seen := make(map[v2.CapabilityDetailCredentialOption]struct{}, len(issue.GetOptions())) + seen := make(map[credentialIssueDescriptorKey]struct{}, len(issue.GetOptions())) + perOption := make(map[v2.CapabilityDetailCredentialOption]int, len(issue.GetOptions())) + preferredPerOption := make(map[v2.CapabilityDetailCredentialOption]int, len(issue.GetOptions())) for _, descriptor := range issue.GetOptions() { if descriptor == nil || descriptor.GetOption() == v2.CapabilityDetailCredentialOption_CAPABILITY_DETAIL_CREDENTIAL_OPTION_UNSPECIFIED { return status.Error(codes.InvalidArgument, "credential issue option descriptor is invalid") @@ -541,6 +543,10 @@ func validateCredentialIssueCapabilityDetails(issue *v2.CredentialDetailsCredent if descriptor.GetSecretResourceTypeId() == "" { return status.Error(codes.InvalidArgument, "credential issue secret resource type is not set") } + if len(descriptor.GetSecretResourceTypeId()) > maxCredentialIssueSecretResourceTypeIDBytes { + return status.Errorf(codes.InvalidArgument, "credential issue secret resource type must be at most %d bytes", + maxCredentialIssueSecretResourceTypeIDBytes) + } if err := validateCredentialIssueDescriptorShape(descriptor); err != nil { return status.Errorf(codes.InvalidArgument, "invalid credential issue option %s: %v", descriptor.GetOption(), err) } @@ -552,17 +558,58 @@ func validateCredentialIssueCapabilityDetails(issue *v2.CredentialDetailsCredent if err := validateIssuanceExpiryCapability(descriptor.GetExpiry()); err != nil { return status.Errorf(codes.InvalidArgument, "invalid credential issue expiry capability: %v", err) } - if _, exists := seen[descriptor.GetOption()]; exists { - return status.Errorf(codes.InvalidArgument, "duplicate credential issue option %s", descriptor.GetOption()) + key := credentialIssueDescriptorKey{ + option: descriptor.GetOption(), + secretResourceTypeID: descriptor.GetSecretResourceTypeId(), + } + if _, exists := seen[key]; exists { + return status.Errorf(codes.InvalidArgument, "duplicate credential issue option %s for secret resource type %q", + descriptor.GetOption(), descriptor.GetSecretResourceTypeId()) + } + seen[key] = struct{}{} + perOption[descriptor.GetOption()]++ + if descriptor.GetPreferred() { + preferredPerOption[descriptor.GetOption()]++ } - seen[descriptor.GetOption()] = struct{}{} } - if _, ok := seen[issue.GetPreferredOption()]; !ok { + if err := validateCredentialIssuePreference(issue.GetOptions(), perOption, preferredPerOption); err != nil { + return err + } + if perOption[issue.GetPreferredOption()] == 0 { return status.Error(codes.InvalidArgument, "preferred credential issue option is not part of the supported options") } return nil } +// validateCredentialIssuePreference enforces that every option resolves to one +// default descriptor. It walks the descriptors in declaration order so the +// error names the first offending option rather than a random one. +func validateCredentialIssuePreference( + descriptors []*v2.CredentialIssueOptionDescriptor, + perOption map[v2.CapabilityDetailCredentialOption]int, + preferredPerOption map[v2.CapabilityDetailCredentialOption]int, +) error { + reported := make(map[v2.CapabilityDetailCredentialOption]struct{}, len(perOption)) + for _, descriptor := range descriptors { + option := descriptor.GetOption() + if _, done := reported[option]; done { + continue + } + reported[option] = struct{}{} + if preferredPerOption[option] > 1 { + return status.Errorf(codes.InvalidArgument, + "credential issue option %s has %d preferred descriptors, expected at most one", + option, preferredPerOption[option]) + } + if perOption[option] > 1 && preferredPerOption[option] == 0 { + return status.Errorf(codes.InvalidArgument, + "credential issue option %s has %d descriptors and none is preferred", + option, perOption[option]) + } + } + return nil +} + func validateCredentialIssueDescriptorShape(descriptor *v2.CredentialIssueOptionDescriptor) error { hasScopes := len(descriptor.GetScopes()) != 0 || descriptor.GetCustomScopesAllowed() hasAudiences := len(descriptor.GetAudiences()) != 0 || descriptor.GetCustomAudiencesAllowed() diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/credential_issue_validation.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/credential_issue_validation.go index 7e2c757b..ee27b777 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/credential_issue_validation.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorbuilder/credential_issue_validation.go @@ -4,6 +4,7 @@ import ( "fmt" "regexp" "slices" + "strconv" "strings" "time" @@ -13,6 +14,56 @@ import ( var credentialIssueRequestIDPattern = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) +// maxCredentialIssueSecretResourceTypeIDBytes bounds secret_resource_type_id on +// both the descriptor and the request. The descriptor's matching proto rules +// never run: nothing on the capabilities or issuance path calls the generated +// Validate(). +const maxCredentialIssueSecretResourceTypeIDBytes = 1024 + +// credentialIssueDescriptorKey identifies one advertised issuance option. A +// credential shape alone cannot: a connector may mint several kinds of +// credential that share a shape and differ only in what they come back as. +type credentialIssueDescriptorKey struct { + option v2.CapabilityDetailCredentialOption + secretResourceTypeID string +} + +// resolveCredentialIssueDescriptor looks up the one descriptor a request's +// CredentialIssueOptions selects: the oneof arm gives the shape, and +// secret_resource_type_id gives the kind within that shape. Both halves are +// required, so the pair always names at most one advertised descriptor. +func resolveCredentialIssueDescriptor( + details *v2.CredentialDetailsCredentialIssue, + options *v2.CredentialIssueOptions, +) (*v2.CredentialIssueOptionDescriptor, error) { + kind := credentialIssueOptionKind(options) + if kind == v2.CapabilityDetailCredentialOption_CAPABILITY_DETAIL_CREDENTIAL_OPTION_UNSPECIFIED { + return nil, fmt.Errorf("unsupported credential option") + } + secretResourceTypeID := options.GetSecretResourceTypeId() + if secretResourceTypeID == "" { + return nil, fmt.Errorf("credential_options.secret_resource_type_id is required") + } + if len(secretResourceTypeID) > maxCredentialIssueSecretResourceTypeIDBytes { + return nil, fmt.Errorf("credential_options.secret_resource_type_id must be at most %d bytes", maxCredentialIssueSecretResourceTypeIDBytes) + } + var advertisedForKind []string + for _, candidate := range details.GetOptions() { + if candidate.GetOption() != kind { + continue + } + if candidate.GetSecretResourceTypeId() == secretResourceTypeID { + return candidate, nil + } + advertisedForKind = append(advertisedForKind, strconv.Quote(candidate.GetSecretResourceTypeId())) + } + if len(advertisedForKind) == 0 { + return nil, fmt.Errorf("credential option %s is not advertised by connector", kind) + } + return nil, fmt.Errorf("credential option %s does not produce secret resource type %q; it produces %s", + kind, secretResourceTypeID, strings.Join(advertisedForKind, ", ")) +} + func credentialIssueOptionKind(options *v2.CredentialIssueOptions) v2.CapabilityDetailCredentialOption { if options == nil { return v2.CapabilityDetailCredentialOption_CAPABILITY_DETAIL_CREDENTIAL_OPTION_UNSPECIFIED @@ -38,19 +89,9 @@ func validateCredentialIssueInput(input *CredentialIssueInput, details *v2.Crede if len(input.RequestID) == 0 || len(input.RequestID) > 128 || !credentialIssueRequestIDPattern.MatchString(input.RequestID) { return nil, fmt.Errorf("request id must be 1..128 characters containing only letters, digits, underscore, or hyphen") } - kind := credentialIssueOptionKind(input.CredentialOptions) - if kind == v2.CapabilityDetailCredentialOption_CAPABILITY_DETAIL_CREDENTIAL_OPTION_UNSPECIFIED { - return nil, fmt.Errorf("unsupported credential option") - } - var descriptor *v2.CredentialIssueOptionDescriptor - for _, candidate := range details.GetOptions() { - if candidate.GetOption() == kind { - descriptor = candidate - break - } - } - if descriptor == nil { - return nil, fmt.Errorf("credential option %s is not advertised by connector", kind) + descriptor, err := resolveCredentialIssueDescriptor(details, input.CredentialOptions) + if err != nil { + return nil, err } if descriptor.GetResourceMode() == v2.CredentialResourceMode_CREDENTIAL_RESOURCE_MODE_UNSPECIFIED { return nil, fmt.Errorf("credential resource mode must be advertised") diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go index afe734f1..be4fee2e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorrunner/runner.go @@ -384,11 +384,6 @@ type eventStreamConfig struct { cursor string } -type syncDifferConfig struct { - baseSyncID string - appliedSyncID string -} - type syncCompactorConfig struct { filePaths []string syncIDs []string @@ -419,7 +414,6 @@ type runnerConfig struct { bulkCreateTicketConfig *bulkCreateTicketConfig listTicketSchemasConfig *listTicketSchemasConfig getTicketConfig *getTicketConfig - syncDifferConfig *syncDifferConfig syncCompactorConfig *syncCompactorConfig skipFullSync bool storageEngine c1zstore.Engine @@ -819,18 +813,6 @@ func WithKeepPreviousSyncC1ZRuntimeOptIn() Option { } } -func WithDiffSyncs(c1zPath string, baseSyncID string, newSyncID string) Option { - return func(ctx context.Context, cfg *runnerConfig) error { - cfg.onDemand = true - cfg.c1zPath = c1zPath - cfg.syncDifferConfig = &syncDifferConfig{ - baseSyncID: baseSyncID, - appliedSyncID: newSyncID, - } - return nil - } -} - func WithSyncCompactor(outputPath string, filePaths []string, syncIDs []string) Option { return func(ctx context.Context, cfg *runnerConfig) error { cfg.onDemand = true @@ -1079,8 +1061,6 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op tm = local.NewGetTicket(ctx, cfg.getTicketConfig.ticketID) case cfg.bulkCreateTicketConfig != nil: tm = local.NewBulkTicket(ctx, cfg.bulkCreateTicketConfig.templatePath) - case cfg.syncDifferConfig != nil: - tm = local.NewDiffer(ctx, cfg.c1zPath, cfg.syncDifferConfig.baseSyncID, cfg.syncDifferConfig.appliedSyncID) case cfg.syncCompactorConfig != nil: c := cfg.syncCompactorConfig if len(c.filePaths) != len(c.syncIDs) { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go b/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go index 3708bf5d..1b081dde 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/connectorstore/connectorstore.go @@ -12,12 +12,10 @@ import ( type SyncType string const ( - SyncTypeFull SyncType = "full" - SyncTypePartial SyncType = "partial" - SyncTypeResourcesOnly SyncType = "resources_only" - SyncTypePartialUpserts SyncType = "partial_upserts" // Diff sync: additions and modifications - SyncTypePartialDeletions SyncType = "partial_deletions" // Diff sync: deletions - SyncTypeAny SyncType = "" + SyncTypeFull SyncType = "full" + SyncTypePartial SyncType = "partial" + SyncTypeResourcesOnly SyncType = "resources_only" + SyncTypeAny SyncType = "" ) var AllSyncTypes = []SyncType{ @@ -25,8 +23,6 @@ var AllSyncTypes = []SyncType{ SyncTypeFull, SyncTypePartial, SyncTypeResourcesOnly, - SyncTypePartialUpserts, - SyncTypePartialDeletions, } // StoreMetadata describes the storage backing a Reader. Returned by @@ -192,21 +188,29 @@ type EntitlementGrantDigestReader interface { // // For 0 <= level <= the native Level (GrantDigest.Level) this folds // the stored leaves — one contiguous scan of the digest keyspace, no - // grant-index scan. For a finer level it falls back to scanning the - // grant index directly (O(grants)) — slower, but it never errors on a - // "too deep" level. The principal-hash carries a bounded number of - // bits, so a level beyond that resolution is served at the maximum - // (you may get fewer than 2^level distinct buckets). found is false - // when no digest exists. + // grant-index scan. For a finer level, up to the principal-hash's + // resolution, it falls back to scanning the grant index directly + // (O(grants)) — slower, but exact. A level outside that resolution + // (a negative level, or one past the implementation's bucket-hash + // width — any level <= the digest's native Level is always in range; + // the Pebble engine exports its full width as DigestBucketHashBits) + // errors rather than silently serving the maximum resolution: + // a caller that placed its own records by hash (e.g. the Pebble + // engine's PrincipalDigestBucket) must get the same bucket set the + // engine reports, not a quietly coarser one. found is false when no + // digest exists. GetEntitlementGrantDigestNodes(ctx context.Context, entitlement *v2.Entitlement, level int) (nodes []GrantDigestNode, found bool, err error) // ScanEntitlementGrantBucket yields every grant in one digest bucket // of the entitlement (see GrantDigestBucket) as a v2.Grant, stopping // early if yield returns false. Bucket Level 0 scans the whole - // entitlement; a Level finer than the bucket-hash resolution is - // clamped (matching GetEntitlementGrantDigestNodes). It reads the - // grant hash index, which exists only on files whose digest was - // built (they are derived together at seal): callers must check + // entitlement; a Level outside the bucket-hash resolution errors + // (matching GetEntitlementGrantDigestNodes) rather than clamping, + // and an Index outside [0, 2^Level) errors rather than wrapping — + // silently folding either coordinate would scan a bucket other than + // the one addressed. It reads the grant hash index, + // which exists only on files whose digest was built (they are + // derived together at seal): callers must check // GetEntitlementGrantDigest first and treat found=false as "scan // unavailable — read the grants directly", not as "no grants". It // yields nothing when there is no active sync or no matching grants. diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/crypto/providers/age/age.go b/vendor/github.com/conductorone/baton-sdk/pkg/crypto/providers/age/age.go index 41f8ef16..3ff7d126 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/crypto/providers/age/age.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/crypto/providers/age/age.go @@ -17,6 +17,24 @@ import ( const EncryptionProviderAge = "baton/age/v1" +// KeyIDForRecipient derives the EncryptedData.key_ids entry for an age recipient. +// +// It returns the lowercase hexadecimal SHA-256 digest of the UTF-8 canonical +// recipient string. This is the single source of truth for the key-ID +// derivation convention: producers set EncryptedData.key_ids to this value, and +// consumers that need to correlate ciphertext with recipient key material must +// call this function rather than reimplementing the derivation. Because both the +// SDK and its consumers import this package, the contract is enforced by shared +// code instead of by prose that can drift. +// +// The recipient must be the canonical recipient string (no surrounding +// whitespace, exactly one recipient); callers that accept untrusted input should +// validate it the same way Encrypt does before deriving a key ID. +func KeyIDForRecipient(recipient string) string { + digest := sha256.Sum256([]byte(recipient)) + return hex.EncodeToString(digest[:]) +} + type RecipientEncryptionProvider struct{} func (p *RecipientEncryptionProvider) ValidateConfig(_ context.Context, conf *v2.EncryptionConfig) error { @@ -42,14 +60,13 @@ func (p *RecipientEncryptionProvider) Encrypt(_ context.Context, conf *v2.Encryp return nil, fmt.Errorf("age: failed to finalize encryption: %w", err) } - keyID := sha256.Sum256([]byte(recipientText)) return v2.EncryptedData_builder{ Provider: EncryptionProviderAge, Name: plaintext.GetName(), Description: plaintext.GetDescription(), Schema: plaintext.GetSchema(), EncryptedBytes: ciphertext.Bytes(), - KeyIds: []string{hex.EncodeToString(keyID[:])}, + KeyIds: []string{KeyIDForRecipient(recipientText)}, }.Build(), nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go index d2c55d23..0a0641d5 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file.go @@ -348,7 +348,9 @@ type c1zOptions struct { disableGrantDigestIndex bool // engine is the storage engine to use for newly created files. - // Reads dispatch on magic byte regardless. Default EngineSQLite. + // Reads dispatch on magic byte regardless. NewStore defaults an + // unset engine to EnginePebble; NewC1ZFile (SQLite-only) normalizes + // unset to EngineSQLite. engine c1zstore.Engine // payloadEncoding controls the v3 envelope payload framing. Only @@ -433,8 +435,10 @@ func WithSyncLimit(limit int) C1ZOption { } // WithEngine selects the storage engine for newly created .c1z files. -// Default is EngineSQLite (v1 format). EnginePebble enables the v3 -// engine. +// Under NewStore the default is EnginePebble (v3 format); EngineSQLite +// selects the legacy v1 engine. NewC1ZFile does not share that default: +// it is the SQLite-only constructor, treats an unset engine as +// EngineSQLite, and rejects a writable EnginePebble request. // // Reading existing files dispatches on the file's magic byte and is // independent of this option. diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_attached.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_attached.go index 6a98a6ae..34d37ede 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_attached.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_attached.go @@ -5,13 +5,11 @@ import ( "database/sql" "errors" "fmt" - "time" reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/uotel" "github.com/doug-martin/goqu/v9" - "github.com/segmentio/ksuid" ) type C1FileAttached struct { @@ -188,275 +186,3 @@ func (c *C1FileAttached) UpdateSync(ctx context.Context, baseSync *reader_v2.Syn return nil } - -// GenerateSyncDiffFromFile compares the old sync (in attached) with the new sync (in main) -// and generates two new syncs in the main database. -// -// IMPORTANT: This assumes main=NEW/compacted and attached=OLD/base: -// - diffTableFromAttached: items in attached (OLD) not in main (NEW) = deletions -// - diffTableFromMain: items in main (NEW) not in attached (OLD) = upserts (additions) -// -// Parameters: -// - oldSyncID: the sync ID in the attached database (OLD/base state) -// - newSyncID: the sync ID in the main database (NEW/compacted state) -// -// Returns (upsertsSyncID, deletionsSyncID, error). -func (c *C1FileAttached) GenerateSyncDiffFromFile(ctx context.Context, oldSyncID string, newSyncID string) (string, string, error) { - if !c.safe { - return "", "", errors.New("database has been detached") - } - - ctx, span := tracer.Start(ctx, "C1FileAttached.GenerateSyncDiffFromFile") - var err error - defer func() { uotel.EndSpanWithError(span, err) }() - - // Verify both source syncs have been backfilled and support diff before - // generating derived syncs. If they haven't, the expansion columns in - // copied grants may be incomplete. - var oldBackfilled, oldDiff int - err = c.file.rawDb.QueryRowContext(ctx, - fmt.Sprintf("SELECT grants_backfilled, supports_diff FROM attached.%s WHERE sync_id = ?", syncRuns.Name()), - oldSyncID, - ).Scan(&oldBackfilled, &oldDiff) - if err != nil { - return "", "", fmt.Errorf("failed to check old sync %s readiness: %w", oldSyncID, err) - } - if oldBackfilled != 1 { - return "", "", fmt.Errorf("old sync %s has not been backfilled (grants_backfilled=%d)", oldSyncID, oldBackfilled) - } - if oldDiff != 1 { - return "", "", fmt.Errorf("old sync %s does not support diff (supports_diff=%d)", oldSyncID, oldDiff) - } - - var newBackfilled, newDiff int - err = c.file.rawDb.QueryRowContext(ctx, - fmt.Sprintf("SELECT grants_backfilled, supports_diff FROM main.%s WHERE sync_id = ?", syncRuns.Name()), - newSyncID, - ).Scan(&newBackfilled, &newDiff) - if err != nil { - return "", "", fmt.Errorf("failed to check new sync %s readiness: %w", newSyncID, err) - } - if newBackfilled != 1 { - return "", "", fmt.Errorf("new sync %s has not been backfilled (grants_backfilled=%d)", newSyncID, newBackfilled) - } - if newDiff != 1 { - return "", "", fmt.Errorf("new sync %s does not support diff (supports_diff=%d)", newSyncID, newDiff) - } - - // Generate unique IDs for the diff syncs - deletionsSyncID := ksuid.New().String() - upsertsSyncID := ksuid.New().String() - - // Start transaction for atomicity - tx, err := c.file.rawDb.BeginTx(ctx, nil) - if err != nil { - return "", "", fmt.Errorf("failed to begin transaction: %w", err) - } - - // Ensure rollback on error - committed := false - defer func() { - if !committed { - _ = tx.Rollback() - } - }() - - now := time.Now().Format(sqliteTimeFormat) - - // Create the deletions sync first (so upserts is "latest") - // Link it to upserts sync bidirectionally - deletionsInsert := c.file.db.Insert(syncRuns.Name()).Rows(goqu.Record{ - "sync_id": deletionsSyncID, - "started_at": now, - "sync_token": "", - "sync_type": connectorstore.SyncTypePartialDeletions, - "parent_sync_id": oldSyncID, - "linked_sync_id": upsertsSyncID, - "supports_diff": 1, - "grants_backfilled": 1, - }) - query, args, err := deletionsInsert.ToSQL() - if err != nil { - return "", "", fmt.Errorf("failed to build deletions sync insert: %w", err) - } - if _, err = tx.ExecContext(ctx, query, args...); err != nil { - return "", "", fmt.Errorf("failed to create deletions sync: %w", err) - } - - // Create the upserts sync, linked to deletions sync - upsertsInsert := c.file.db.Insert(syncRuns.Name()).Rows(goqu.Record{ - "sync_id": upsertsSyncID, - "started_at": now, - "sync_token": "", - "sync_type": connectorstore.SyncTypePartialUpserts, - "parent_sync_id": oldSyncID, - "linked_sync_id": deletionsSyncID, - "supports_diff": 1, - "grants_backfilled": 1, - }) - query, args, err = upsertsInsert.ToSQL() - if err != nil { - return "", "", fmt.Errorf("failed to build upserts sync insert: %w", err) - } - if _, err = tx.ExecContext(ctx, query, args...); err != nil { - return "", "", fmt.Errorf("failed to create upserts sync: %w", err) - } - - // Process each table - // main=NEW, attached=OLD - // - diffTableFromAttachedTx finds items in OLD not in NEW = deletions - // - diffTableFromMainTx finds items in NEW not in OLD or modified = upserts - tables := []string{"v1_resource_types", "v1_resources", "v1_entitlements", "v1_grants"} - for _, tableName := range tables { - if err := c.diffTableFromAttachedTx(ctx, tx, tableName, oldSyncID, newSyncID, deletionsSyncID); err != nil { - return "", "", fmt.Errorf("failed to generate deletions for %s: %w", tableName, err) - } - if err := c.diffTableFromMainTx(ctx, tx, tableName, oldSyncID, newSyncID, upsertsSyncID); err != nil { - return "", "", fmt.Errorf("failed to generate upserts for %s: %w", tableName, err) - } - } - - // End the syncs (deletions first, then upserts) - endedAt := time.Now().Format(sqliteTimeFormat) - - endDeletions := c.file.db.Update(syncRuns.Name()). - Set(goqu.Record{"ended_at": endedAt}). - Where(goqu.C("sync_id").Eq(deletionsSyncID), goqu.C("ended_at").IsNull()) - query, args, err = endDeletions.ToSQL() - if err != nil { - return "", "", fmt.Errorf("failed to build end deletions sync: %w", err) - } - if _, err = tx.ExecContext(ctx, query, args...); err != nil { - return "", "", fmt.Errorf("failed to end deletions sync: %w", err) - } - - endUpserts := c.file.db.Update(syncRuns.Name()). - Set(goqu.Record{"ended_at": endedAt}). - Where(goqu.C("sync_id").Eq(upsertsSyncID), goqu.C("ended_at").IsNull()) - query, args, err = endUpserts.ToSQL() - if err != nil { - return "", "", fmt.Errorf("failed to build end upserts sync: %w", err) - } - if _, err = tx.ExecContext(ctx, query, args...); err != nil { - return "", "", fmt.Errorf("failed to end upserts sync: %w", err) - } - - // Commit transaction - if err = tx.Commit(); err != nil { - return "", "", fmt.Errorf("failed to commit transaction: %w", err) - } - committed = true - c.file.dbUpdated.Store(true) - - return upsertsSyncID, deletionsSyncID, nil -} - -// diffTableFromAttachedTx finds items in attached (OLD) that don't exist in main (NEW). -// These are DELETIONS - items that existed before but no longer exist. -// Uses the provided transaction. -func (c *C1FileAttached) diffTableFromAttachedTx(ctx context.Context, tx *sql.Tx, tableName string, oldSyncID string, newSyncID string, targetSyncID string) error { - columns, err := c.getTableColumns(ctx, tx, tableName) - if err != nil { - return err - } - - // Build column lists - columnList := "" - selectList := "" - for i, col := range columns { - if i > 0 { - columnList += ", " - selectList += ", " - } - qcol := quoteIdentifier(col) - columnList += qcol - if col == "sync_id" { - selectList += "? as " + qcol - } else { - selectList += qcol - } - } - - // Insert items from attached (OLD) that don't exist in main (NEW) - // oldSyncID is in attached, newSyncID is in main - //nolint:gosec // table names are from hardcoded list; column names are validated - query := fmt.Sprintf(` - INSERT INTO main.%s (%s) - SELECT %s - FROM attached.%s AS a - WHERE a.sync_id = ? - AND NOT EXISTS ( - SELECT 1 FROM main.%s AS m - WHERE m.external_id = a.external_id AND m.sync_id = ? - ) - `, tableName, columnList, selectList, tableName, tableName) - - _, err = tx.ExecContext(ctx, query, targetSyncID, oldSyncID, newSyncID) - return err -} - -// diffTableFromMainTx finds items in main (NEW) that are new or modified compared to attached (OLD). -// These are UPSERTS - items that are new or have changed. -// Uses the provided transaction. -func (c *C1FileAttached) diffTableFromMainTx(ctx context.Context, tx *sql.Tx, tableName string, oldSyncID string, newSyncID string, targetSyncID string) error { - columns, err := c.getTableColumns(ctx, tx, tableName) - if err != nil { - return err - } - - // Build column lists - columnList := "" - selectList := "" - for i, col := range columns { - if i > 0 { - columnList += ", " - selectList += ", " - } - qcol := quoteIdentifier(col) - columnList += qcol - if col == "sync_id" { - selectList += "? as " + qcol - } else { - selectList += qcol - } - } - - // Insert items from main (NEW) that are: - // 1. Not in attached (OLD) - additions - // 2. In attached but with different data - modifications - // newSyncID is in main, oldSyncID is in attached - // - // For grants, we also compare the expansion column since GrantExpandable - // annotation is stored separately from data. - var dataCompare string - if tableName == grants.Name() { - // For grants: compare both data AND expansion columns. - // Use IFNULL to handle NULL expansion values. - dataCompare = "(a.data != m.data OR IFNULL(a.expansion, X'') != IFNULL(m.expansion, X''))" - } else { - dataCompare = "a.data != m.data" - } - - //nolint:gosec // table names are from hardcoded list; column names are validated - query := fmt.Sprintf(` - INSERT INTO main.%s (%s) - SELECT %s - FROM main.%s AS m - WHERE m.sync_id = ? - AND ( - NOT EXISTS ( - SELECT 1 FROM attached.%s AS a - WHERE a.external_id = m.external_id AND a.sync_id = ? - ) - OR EXISTS ( - SELECT 1 FROM attached.%s AS a - WHERE a.external_id = m.external_id - AND a.sync_id = ? - AND %s - ) - ) - `, tableName, columnList, selectList, tableName, tableName, tableName, dataCompare) - - _, err = tx.ExecContext(ctx, query, targetSyncID, newSyncID, oldSyncID, oldSyncID) - return err -} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go index bc5d5c14..7e1281c6 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1file_store.go @@ -299,7 +299,7 @@ func (s c1FileSyncMeta) LatestFullSync(ctx context.Context) (*c1zstore.SyncRun, } // LatestFinishedSyncOfAnyType implements SyncMeta. Returns the most-recent -// finished sync of any type (including diff types), or nil if none. +// finished sync of any type, or nil if none. func (s c1FileSyncMeta) LatestFinishedSyncOfAnyType(ctx context.Context) (*c1zstore.SyncRun, error) { run, err := s.c.getFinishedSync(ctx, 0, connectorstore.SyncTypeAny) if err != nil { @@ -354,11 +354,6 @@ func (f c1FileFileOps) CopyIsolateSync(ctx context.Context, outPath string, sync return f.c.CopyIsolateSync(ctx, outPath, syncID, c1fOpts...) } -// GenerateSyncDiff implements FileOps. Direct passthrough. -func (f c1FileFileOps) GenerateSyncDiff(ctx context.Context, baseSyncID, appliedSyncID string) (string, error) { - return f.c.GenerateSyncDiff(ctx, baseSyncID, appliedSyncID) -} - type c1FileSessionStore struct{ c *C1File } func (s c1FileSessionStore) Get(ctx context.Context, key string, opt ...sessions.SessionStoreOption) ([]byte, bool, error) { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go index 8aa50210..3a5f51b5 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/c1zstore.go @@ -28,7 +28,7 @@ import ( // // - Grant operations with expansion-aware semantics, accessed via Grants(). // - Sync-run metadata operations, accessed via SyncMeta(). -// - File-level operations (clone, diff), accessed via FileOps(). +// - File-level operations (clone), accessed via FileOps(). // // Implementations: // @@ -61,3 +61,17 @@ type Store interface { SessionStore() sessions.SessionStore } + +// GrantGenerationDigest binds derived metadata to the exact grant generation +// stored in an artifact. +type GrantGenerationDigest struct { + Hash []byte + Count int64 + ABIVersion uint32 +} + +// GrantGenerationDigestReader is implemented by stores that persist an exact +// whole-file grant digest at seal time. +type GrantGenerationDigestReader interface { + GrantGenerationDigest(ctx context.Context) (GrantGenerationDigest, bool, error) +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/cleanup_policy.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/cleanup_policy.go index 646cabeb..3f86d3a8 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/cleanup_policy.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/cleanup_policy.go @@ -22,9 +22,9 @@ const defaultCleanupSyncLimit = 2 // considered "in flight" and must never be pruned). // - currentSyncID is skipped when non-empty (the actively-open sync // is also off-limits). -// - Candidates are bucketed by Type into fullSyncs, partials, and -// diff syncs. SyncTypeFull and any unrecognized type go into -// fullSyncs (matches the SQLite default branch). +// - Candidates are bucketed by Type into fullSyncs and partials. +// SyncTypeFull and any unrecognized type go into fullSyncs +// (matches the SQLite default branch). // - syncLimit is the number of *additional* full syncs to retain // beyond the current one. The caller has already decremented for // a running sync (see ResolveCleanupSyncLimit), so this function @@ -32,9 +32,6 @@ const defaultCleanupSyncLimit = 2 // oldest overflow is selected for deletion. // - Once the earliest-kept full sync is established, partials that // ended before that sync started are selected for deletion. -// - When more than two diff syncs (partial_upserts / partial_deletions) -// exist, only the most recent diff sync and its linked pair are -// retained; everything else is selected. // // Order matters: callers must pass candidates in oldest-first order // so "drop the oldest overflow" trims the right end. SQLite supplies @@ -44,7 +41,6 @@ const defaultCleanupSyncLimit = 2 func SelectSyncsToDelete(candidates []SyncRun, currentSyncID string, syncLimit int) []string { var fullSyncs []SyncRun var partials []SyncRun - var diffSyncs []SyncRun for _, sr := range candidates { if sr.EndedAt == nil || sr.ID == currentSyncID { @@ -53,8 +49,6 @@ func SelectSyncsToDelete(candidates []SyncRun, currentSyncID string, syncLimit i switch sr.Type { case connectorstore.SyncTypePartial, connectorstore.SyncTypeResourcesOnly: partials = append(partials, sr) - case connectorstore.SyncTypePartialUpserts, connectorstore.SyncTypePartialDeletions: - diffSyncs = append(diffSyncs, sr) default: fullSyncs = append(fullSyncs, sr) } @@ -86,31 +80,6 @@ func SelectSyncsToDelete(candidates []SyncRun, currentSyncID string, syncLimit i } } - // Diff syncs: keep latest + its linked partner; drop the rest. - // Mirrors the SQLite branch at sync_runs.go:884-931. The - // "diffSyncs > 2" guard preserves the historical no-op behavior - // for small histories — we don't prune until there's enough to - // be worth touching. - if len(diffSyncs) > 2 { - syncByID := make(map[string]SyncRun, len(diffSyncs)) - for _, ds := range diffSyncs { - syncByID[ds.ID] = ds - } - latestDiff := diffSyncs[len(diffSyncs)-1] - keepIDs := map[string]struct{}{latestDiff.ID: {}} - if latestDiff.LinkedSyncID != "" { - if _, ok := syncByID[latestDiff.LinkedSyncID]; ok { - keepIDs[latestDiff.LinkedSyncID] = struct{}{} - } - } - for _, ds := range diffSyncs { - if _, keep := keepIDs[ds.ID]; keep { - continue - } - toDelete = append(toDelete, ds.ID) - } - } - return toDelete } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/engine.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/engine.go index 833fa7cd..9fb8a684 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/engine.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/engine.go @@ -13,12 +13,15 @@ import ( type Engine string const ( - // EngineSQLite is the default engine: the v1 .c1z format backed by - // a zstd-compressed SQLite database. Connectors use this; backend - // infra can opt out. + // EngineSQLite is the legacy v1 engine: the v1 .c1z format backed by + // a zstd-compressed SQLite database. Callers opt into it via + // WithEngine; the NewStore default is EnginePebble. (The SQLite-only + // NewC1ZFile constructor is the exception: it treats an unset engine + // as EngineSQLite.) EngineSQLite Engine = "sqlite" - // EnginePebble is the v3 engine: a Pebble LSM wrapped in the v3 + // EnginePebble is the v3 engine and the NewStore default when + // callers do not specify one: a Pebble LSM wrapped in the v3 // envelope. This is the in-process identity AND the value callers // select with (the --storage-engine flag and the gRPC sync-task // field both pass "pebble"); it must stay "pebble" for those diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/file_ops.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/file_ops.go index dd219b7b..9bee5546 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/file_ops.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/file_ops.go @@ -18,11 +18,6 @@ type FileOps interface { // the target sync row-by-row. It is the optimized isolation step for large // files; the output contains only the target sync and is schema-normalized. CopyIsolateSync(ctx context.Context, outPath string, syncID string, opts ...CloneSyncOption) error - - // GenerateSyncDiff computes the diff between two existing sync runs - // in this same file and writes the delta as a new SyncTypePartial - // sync. Returns the new sync's id. Used by the local differ CLI. - GenerateSyncDiff(ctx context.Context, baseSyncID, appliedSyncID string) (diffSyncID string, err error) } // CloneSyncOptions carries the engine-neutral knobs for FileOps.CloneSync. diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/sync_meta.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/sync_meta.go index 1eb1d5c3..8ea4c4f0 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/sync_meta.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore/sync_meta.go @@ -15,9 +15,10 @@ import ( // All methods are callable without an active sync. type SyncMeta interface { // MarkSyncSupportsDiff sets the supports_diff flag on the given sync. - // Called by pkg/sync.parallelSyncer after graph construction to signal - // that the sync run has SQL-layer grant metadata populated and diff - // consumers may rely on it. + // Called by pkg/sync.parallelSyncer when data collection completes to + // signal that the sync run has SQL-layer grant metadata populated. + // The name is historical (the marker once gated diff-sync generation); + // today it gates `baton rollback-expansion`. MarkSyncSupportsDiff(ctx context.Context, syncID string) error // LatestFullSync returns the most-recently-finished SyncTypeFull sync @@ -25,9 +26,8 @@ type SyncMeta interface { LatestFullSync(ctx context.Context) (*SyncRun, error) // LatestFinishedSyncOfAnyType returns the most-recently-finished sync - // of any type (including diff types), or nil if none exists. Used by - // tooling that wants to inspect whatever sync finished last regardless - // of type. + // of any type, or nil if none exists. Used by tooling that wants to + // inspect whatever sync finished last regardless of type. LatestFinishedSyncOfAnyType(ctx context.Context) (*SyncRun, error) // Stats returns a map of table-name to row-count for the given sync. @@ -66,8 +66,7 @@ type IngestInvariantVerificationWriter interface { // sync_runs schema. // // Callers typically only read ID, Type, and the timestamps; the rest is -// included for completeness and for use by tooling (e.g. sync-diff -// pipelines need ParentSyncID and LinkedSyncID). +// included for completeness and for use by tooling. type SyncRun struct { ID string StartedAt *time.Time @@ -75,12 +74,24 @@ type SyncRun struct { SyncToken string Type connectorstore.SyncType ParentSyncID string - LinkedSyncID string SupportsDiff bool + Compacted bool Stats *reader_v2.SyncStats IngestInvariantVerification } +// UsableAsReplaySource reports whether this sync's upstream validators can +// describe its contents. Compaction is a keep-newer merge rather than a +// connector snapshot, so compacted and non-full syncs must be treated as cold +// cache inputs. This checks run metadata only; callers must separately require +// a storage engine that implements source-cache replay and authoritatively +// persists compaction provenance. SQLite currently does neither, so its +// zero-value Compacted field is not evidence that an artifact was never +// compacted. +func (r SyncRun) UsableAsReplaySource() bool { + return r.Type == connectorstore.SyncTypeFull && !r.Compacted +} + // IngestInvariantVerification is persisted provenance for a successful // post-collection invariant pass. Generation identifies the verifier // contract, Coverage lists the invariant IDs that actually ran, and Mode diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/diff.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/diff.go deleted file mode 100644 index c991b585..00000000 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/diff.go +++ /dev/null @@ -1,124 +0,0 @@ -package dotc1z - -import ( - "context" - "fmt" - "strings" - - "github.com/conductorone/baton-sdk/pkg/connectorstore" - "github.com/doug-martin/goqu/v9" - "github.com/segmentio/ksuid" -) - -func (c *C1File) GenerateSyncDiff(ctx context.Context, baseSyncID string, appliedSyncID string) (string, error) { - if c.readOnly { - return "", ErrReadOnly - } - - // Validate that both sync runs exist - baseSync, err := c.getSync(ctx, baseSyncID) - if err != nil { - return "", err - } - if baseSync == nil { - return "", fmt.Errorf("generate-diff: base sync not found") - } - - newSync, err := c.getSync(ctx, appliedSyncID) - if err != nil { - return "", err - } - if newSync == nil { - return "", fmt.Errorf("generate-diff: new sync not found") - } - - // Generate a new unique ID for the diff sync - diffSyncID := ksuid.New().String() - - if err := c.insertSyncRun(ctx, diffSyncID, connectorstore.SyncTypePartial, baseSyncID); err != nil { - return "", err - } - - for _, t := range allTableDescriptors { - if strings.Contains(t.Name(), syncRunsTableName) { - continue - } - - q, args, err := c.diffTableQuery(t, baseSyncID, appliedSyncID, diffSyncID) - if err != nil { - return "", err - } - if q == "" { - continue - } - _, err = c.db.ExecContext(ctx, q, args...) - if err != nil { - return "", err - } - c.dbUpdated.Store(true) - } - - if err := c.endSyncRun(ctx, diffSyncID); err != nil { - return "", err - } - - return diffSyncID, nil -} - -func (c *C1File) diffTableQuery(table tableDescriptor, baseSyncID, appliedSyncID, newSyncID string) (string, []any, error) { - // Define the columns to select based on the table name - columns := []interface{}{ - "external_id", - "data", - "sync_id", - "discovered_at", - } - - tableName := table.Name() - // Add table-specific columns - switch { - case strings.Contains(tableName, sessionStoreTableName): - // caching is not relevant to diffs. - return "", nil, nil - case strings.Contains(tableName, resourcesTableName): - columns = append(columns, "resource_type_id", "parent_resource_type_id", "parent_resource_id") - case strings.Contains(tableName, resourceTypesTableName): - // Nothing new to add here - case strings.Contains(tableName, grantsTableName): - columns = append(columns, "resource_type_id", "resource_id", "entitlement_id", "principal_resource_type_id", "principal_resource_id") - case strings.Contains(tableName, entitlementsTableName): - columns = append(columns, "resource_type_id", "resource_id") - case strings.Contains(tableName, assetsTableName): - columns = append(columns, "content_type") - } - - // Build the subquery to find external_ids in the base sync - subquery := c.db.Select("external_id"). - From(tableName). - Where(goqu.C("sync_id").Eq(baseSyncID)) - - queryColumns := []interface{}{} - for _, col := range columns { - if col == "sync_id" { //nolint:goconst,nolintlint // ... - queryColumns = append(queryColumns, goqu.L(fmt.Sprintf("'%s' as sync_id", newSyncID))) - continue - } - queryColumns = append(queryColumns, col) - } - - // Build the main query to select records from newSyncID that don't exist in baseSyncID - query := c.db.Insert(tableName). - Cols(columns...). - Prepared(true). - FromQuery( - c.db.Select(queryColumns...). - From(tableName). - Where( - goqu.C("sync_id").Eq(appliedSyncID), - goqu.C("external_id").NotIn(subquery), - ), - ) - - // Generate the SQL and args - return query.ToSQL() -} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go index 23048684..45a46fd8 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter.go @@ -23,6 +23,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb" + "github.com/conductorone/baton-sdk/pkg/sourcecache" ) // This file is the Engine's connectorstore face: the sync lifecycle @@ -334,6 +335,14 @@ func (e *Engine) endSyncFinalize(ctx context.Context, existing *v3.SyncRunRecord return fmt.Errorf("EndSync: repair grant digests: %w", err) } } + // Seal the source-cache manifest's per-scope row counts BEFORE the + // ended_at stamp (CO-004): the counts must be provably present in any + // artifact that carries the finished verdict, because replay preflight + // hard-fails on a manifest entry without one. A crash in between + // leaves the sync unfinished and the resumed EndSync recounts. + if err := e.sealSourceCacheRowCounts(ctx); err != nil { + return fmt.Errorf("EndSync: seal source cache row counts: %w", err) + } // Preserve all provenance fields while adding the lifecycle stamp. updated := proto.Clone(existing).(*v3.SyncRunRecord) updated.SetEndedAt(timestamppb.Now()) @@ -406,12 +415,23 @@ func (e *Engine) PutGrants(ctx context.Context, grants ...*v2.Grant) error { return ErrNoCurrentSync } records := translateGrants(syncID, grants) + stampSourceScope(ctx, records, func(r *v3.GrantRecord, scope string) { r.SetSourceScopeKey(scope) }) if err := e.PutGrantRecords(ctx, records...); err != nil { return fmt.Errorf("PutGrants: %w", err) } return nil } +func stampSourceScope[T any](ctx context.Context, records []T, set func(T, string)) { + scope := sourcecache.ScopeFromContext(ctx) + if scope == "" { + return + } + for _, record := range records { + set(record, scope) + } +} + // UnsafePutUniqueGrants writes grants on the trusted-import path: records // are encoded in parallel and written unconditionally, with no read-before-write // and no dedup pass. Do not use it for live connector output. The destination @@ -569,6 +589,7 @@ func (e *Engine) PutResources(ctx context.Context, resources ...*v2.Resource) er } records = append(records, rec) } + stampSourceScope(ctx, records, func(r *v3.ResourceRecord, scope string) { r.SetSourceScopeKey(scope) }) if err := e.PutResourceRecords(ctx, records...); err != nil { return fmt.Errorf("PutResources: %w", err) } @@ -596,6 +617,7 @@ func (e *Engine) PutEntitlements(ctx context.Context, entitlements ...*v2.Entitl } records = append(records, rec) } + stampSourceScope(ctx, records, func(r *v3.EntitlementRecord, scope string) { r.SetSourceScopeKey(scope) }) if err := e.PutEntitlementRecords(ctx, records...); err != nil { return fmt.Errorf("PutEntitlements: %w", err) } @@ -1041,10 +1063,6 @@ func v2SyncTypeToV3(t connectorstore.SyncType) v3.SyncType { return v3.SyncType_SYNC_TYPE_PARTIAL case connectorstore.SyncTypeResourcesOnly: return v3.SyncType_SYNC_TYPE_RESOURCES_ONLY - case connectorstore.SyncTypePartialUpserts: - return v3.SyncType_SYNC_TYPE_PARTIAL_UPSERTS - case connectorstore.SyncTypePartialDeletions: - return v3.SyncType_SYNC_TYPE_PARTIAL_DELETIONS default: return v3.SyncType_SYNC_TYPE_UNSPECIFIED } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_diff.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_diff.go deleted file mode 100644 index 26b81937..00000000 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_diff.go +++ /dev/null @@ -1,21 +0,0 @@ -package pebble - -import ( - "context" - "errors" -) - -// ErrDiffUnsupported is returned by the Pebble v3 engine's -// GenerateSyncDiff. A v3 c1z holds exactly one sync by contract, so the -// precondition GenerateSyncDiff needs — two ended syncs (base + applied) -// co-resident in one file — can never be satisfied. Diffs must be -// computed a layer up, across two separate c1z files. -var ErrDiffUnsupported = errors.New("pebble v3 engine: GenerateSyncDiff is unsupported (single-sync contract)") - -// generateSyncDiff is unsupported on the single-sync Pebble engine; see -// ErrDiffUnsupported. The previous additions-only set-difference -// implementation was removed when the keyspace dropped its sync_id -// region (a second sync can no longer coexist with the base). -func generateSyncDiff(_ context.Context, _ *Adapter, _, _ string) (string, error) { - return "", ErrDiffUnsupported -} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_file_ops.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_file_ops.go index 4e2bf73a..cdef69f9 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_file_ops.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_file_ops.go @@ -8,9 +8,7 @@ import ( // FileOps returns the FileOps sub-store backed by the Pebble // adapter. Implements c1zstore.Store.FileOps(). CloneSync materializes -// the single sync's data into a fresh c1z (used by `baton clone`); -// GenerateSyncDiff is unsupported (single-sync contract — see -// ErrDiffUnsupported). +// the single sync's data into a fresh c1z (used by `baton clone`). func (e *Engine) FileOps() c1zstore.FileOps { return pebbleFileOps{e: e, encoding: c1zstore.PayloadEncodingTarZstd} } @@ -45,10 +43,3 @@ func (f pebbleFileOps) CloneSync(ctx context.Context, outPath string, syncID str func (f pebbleFileOps) CopyIsolateSync(ctx context.Context, outPath string, syncID string, opts ...c1zstore.CloneSyncOption) error { return cloneSync(ctx, f.e, f.encoding, outPath, syncID, opts...) } - -// GenerateSyncDiff is unsupported on the Pebble v3 engine — a c1z -// holds exactly one sync by contract, so base + applied syncs can't be -// co-resident in one file. Always returns ErrDiffUnsupported. -func (f pebbleFileOps) GenerateSyncDiff(ctx context.Context, baseSyncID, appliedSyncID string) (string, error) { - return generateSyncDiff(ctx, f.e, baseSyncID, appliedSyncID) -} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go index de414443..ebbb0a77 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_grants_store.go @@ -17,7 +17,7 @@ import ( // Grants returns the GrantStore implementation backed by the Pebble // adapter. Implements c1zstore.Store.Grants(); used by the -// expander, the c1-side fileClientWrapper, and the differ. +// expander and the c1-side fileClientWrapper. // // needs_expansion is populated at PutGrants time: V2GrantToV3 extracts // the GrantExpandable annotation and sets NeedsExpansion, which keys the @@ -220,6 +220,10 @@ func (g pebbleGrantStore) translateExpanded(syncID string, grants []*v2.Grant) [ // because the caller left a residual GrantExpandable annotation. newRec.SetExpansion(nil) newRec.SetNeedsExpansion(false) + // Existing direct grants recover their prior stamp in + // PutExpandedGrantRecords; brand-new expander-derived rows never + // belong to a connector source scope. + newRec.SetSourceScopeKey("") merged = append(merged, newRec) } return merged diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_reader.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_reader.go index d0782dd5..882c698f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_reader.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/adapter_reader.go @@ -719,13 +719,18 @@ func (e *Engine) GetEntitlementGrantDigest(ctx context.Context, ent *v2.Entitlem // grant-digest rollup nodes at the requested level (2^level buckets; // level 0 = the root). For 0 <= level <= the digest's native level it // folds the stored leaves — one scan of the digest keyspace. For a finer -// level it scans the grant index directly (O(grants)) instead of -// erroring; the level is clamped to the bucket-hash resolution -// (digestMaxWidthBits). +// level, up to digestMaxWidthBits, it scans the grant index directly +// (O(grants)) instead. A level outside [0, digestMaxWidthBits] errors: +// the bucket hash carries no more resolution than digestMaxWidthBits, so +// silently clamping would report buckets a caller's own precomputed +// index (see PrincipalDigestBucket) does not agree with. func (e *Engine) GetEntitlementGrantDigestNodes(ctx context.Context, ent *v2.Entitlement, level int) ([]connectorstore.GrantDigestNode, bool, error) { if level < 0 { return nil, false, fmt.Errorf("pebble: negative grant-digest level %d", level) } + if level > digestMaxWidthBits { + return nil, false, fmt.Errorf("pebble: grant-digest level %d exceeds bucket-hash resolution %d", level, digestMaxWidthBits) + } syncID, err := e.resolveActiveSyncForReader(ctx, nil) if err != nil { return nil, false, err @@ -746,11 +751,10 @@ func (e *Engine) GetEntitlementGrantDigestNodes(ctx context.Context, ent *v2.Ent if level == 0 { return []connectorstore.GrantDigestNode{{Index: 0, Hash: root.Hash, Count: root.Count}}, true, nil } - // The bucket hash carries at most digestMaxWidthBits of resolution; - // a finer level can't address more buckets, so clamp. - bits := min(level, digestMaxWidthBits) - // At or below the stored width, fold the digest leaves (cheap). Finer - // than what we stored, scan the grant index to compute the rollup. + // level is already bounded to [0, digestMaxWidthBits] above. At or + // below the stored width, fold the digest leaves (cheap); finer than + // what we stored, scan the grant index to compute the rollup. + bits := level partition := digestPartitionForEntitlement(id) var folded []foldedBucket if bits <= root.Bits { @@ -775,13 +779,27 @@ func (e *Engine) GetEntitlementGrantDigestNodes(ctx context.Context, ent *v2.Ent // ScanEntitlementGrantBucket implements // connectorstore.EntitlementGrantDigestReader. It yields every grant in // the given digest bucket of the entitlement, translated to v2.Grant. -// Bucket Level 0 scans the whole entitlement; a finer Level is clamped -// to the bucket-hash resolution. Yields nothing when there is no active -// sync or a bare entitlement id resolves to nothing. +// Bucket Level 0 scans the whole entitlement. A Level outside +// [0, digestMaxWidthBits] errors rather than clamping to the bucket-hash +// resolution, and an Index outside [0, 2^Level) errors rather than +// wrapping to its low Level bits: either kind of silent folding would +// scan a bucket other than the one the caller addressed (see +// PrincipalDigestBucket, which only builds in-range buckets). Yields +// nothing when there is no active sync or a bare entitlement id +// resolves to nothing. func (e *Engine) ScanEntitlementGrantBucket(ctx context.Context, ent *v2.Entitlement, bucket connectorstore.GrantDigestBucket, yield func(*v2.Grant) bool) error { if bucket.Level < 0 { return fmt.Errorf("pebble: negative grant-digest level %d", bucket.Level) } + if bucket.Level > digestMaxWidthBits { + return fmt.Errorf("pebble: grant-digest level %d exceeds bucket-hash resolution %d", bucket.Level, digestMaxWidthBits) + } + // Level 0 ignores Index (whole-entitlement scan) per the + // GrantDigestBucket contract; past that, bucketBounds would shift an + // oversized index's high bits away and scan Index mod 2^Level. + if bucket.Level > 0 && uint64(bucket.Index) >= 1<> (16 - bits)" shifts in +// bucketOfHash / foldedLeafBuckets / computeBucketsAtWidth safe: +// growing digestMaxWidthBits without digestLeafPrefixLen would leave +// bits able to exceed 16 and panic on a negative shift at read time. +const _ uint = digestMaxWidthBits - digestBucketHashLen*8 +const _ uint = digestBucketHashLen*8 - digestMaxWidthBits +const _ uint = digestMaxWidthBits - digestLeafPrefixLen*8 +const _ uint = digestLeafPrefixLen*8 - digestMaxWidthBits + // Node-key levels: the root is level 0 (empty prefix); the single leaf // level is 1 (digestLeafPrefixLen-byte prefix). See encodeDigestNodeKey. const ( diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine.go index 6e702cf0..acf267ee 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/engine.go @@ -13,6 +13,7 @@ import ( "github.com/cockroachdb/pebble/v2" "github.com/cockroachdb/pebble/v2/vfs" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" "google.golang.org/protobuf/proto" v2pb "github.com/conductorone/baton-sdk/pb/c1/connector/v2" @@ -88,8 +89,9 @@ type Engine struct { // call only" — subsequent calls in the same fresh sync must // still read-before-write to clean up cross-call duplicate index // entries. - freshGrantsEmpty bool - freshResourcesEmpty bool + freshGrantsEmpty bool + freshEntitlementsEmpty bool + freshResourcesEmpty bool // writeWG tracks in-flight writes. Incremented at the start of // every Writer method, decremented in defer. @@ -170,8 +172,8 @@ type Engine struct { // save/close (checkpoint + envelope encode) never benefits from either. // Binding a sync again (SetCurrentSync / MarkFreshSync) unseals and // resumes compactions. Sync-run metadata writes (PutSyncRunRecord and - // friends) are exempt — callers legitimately stamp ended_at overrides, - // diff links, and supports_diff markers on a finished sync. Without this + // friends) are exempt — callers legitimately stamp ended_at overrides + // and supports_diff markers on a finished sync. Without this // state the "no writes while compactions are paused" invariant was // convention only, and a caller that kept writing after EndSync would // silently accumulate L0 until pebble stalled writes at @@ -305,6 +307,74 @@ func Open(ctx context.Context, dir string, opts ...Option) (*Engine, error) { _ = e.Close() return nil, err } + // Arm the mutation-path source-scope index obligations iff the file + // actually holds by_source_scope entries (bounded seeks, same + // contract as the digest probe): scope-free stores keep the exact + // pre-scope write cost. Runs BEFORE migrations so any migration + // staging typed record ops sees a derived gate, not the false + // default; a migration that backfills by_source_scope entries must + // itself re-probe or arm (see the indexMigrations registry doc). + if err := e.db.ProbeSourceScopeMayExist(); err != nil { + _ = e.Close() + return nil, err + } + // Poison events (CO-015) are always actionable — the scope re-fetches + // cold next sync, and persistent overlap means the connector's + // partitioning is wrong — but NOT rare per sync in the mis-partitioned + // case: batch-level staging dedups only within one RecordBatch, and + // batches re-mint per chunk, so a persistently overlapping scope (the + // external-principal reconciliation shape included) would otherwise + // warn once per 10k-row chunk. Dedup here, per (kind, scope) per open: + // one warning per poisoned scope per artifact is the diagnostic + // signal; the durable marker itself stays idempotent per batch. + // The dedup set is a pure log cache in the connector-controlled + // scope dimension, so it is bounded like every other scope-scale + // allocation: past the cap, one notice and further UNSEEN scopes go + // unlogged (already-seen scopes stay deduplicated) — thousands of + // distinct poisoned scopes is a partitioning pathology where + // per-scope lines stop adding signal, and the durable markers still + // record every scope for direct inspection. Logger captured at open + // — staging batches have no ctx at commit time. Mutex because + // batches from different callers may commit concurrently. + const poisonLogSetCap = 4096 + poisonLogger := ctxzap.Extract(ctx) + var poisonLogMu sync.Mutex + poisonLogged := make(map[[2]string]struct{}) + poisonLogCapped := false + e.db.SetPoisonObserver(func(ev rawdb.PoisonEvent) { + // Bound resolved at event time so the test seam (set after open, + // before any mutation commits) can shrink it; events deliver on + // the committing goroutine, so the read is ordered after the + // test's write. + bound := poisonLogSetCap + if e.test.poisonLogSetCap > 0 { + bound = e.test.poisonLogSetCap + } + seen := [2]string{ev.RowKind, ev.ScopeKey} + poisonLogMu.Lock() + if _, dup := poisonLogged[seen]; dup { + poisonLogMu.Unlock() + return + } + if len(poisonLogged) >= bound { + notice := !poisonLogCapped + poisonLogCapped = true + poisonLogMu.Unlock() + if notice { + poisonLogger.Warn("pebble: further source-cache poison warnings suppressed — distinct poisoned scopes exceeded the log-dedup bound; durable poison markers still record every scope", + zap.Int("bound", bound), + ) + } + return + } + poisonLogged[seen] = struct{}{} + poisonLogMu.Unlock() + poisonLogger.Warn("pebble: source-cache scope poisoned — refused as a replay source for this artifact", + zap.String("row_kind", ev.RowKind), + zap.String("scope_key", ev.ScopeKey), + zap.String("cause", ev.Cause), + ) + }) // Run secondary-index migrations before returning. Migrations // are skipped for read-only opens (the on-disk file is // immutable, so we'd error out trying to backfill). @@ -375,11 +445,24 @@ func (e *Engine) bindCurrentSync(syncID string) error { e.currentSync = idBytes e.freshSync = false e.freshGrantsEmpty = false + e.freshEntitlementsEmpty = false e.freshResourcesEmpty = false e.currentSyncMu.Unlock() // Binding a sync means more writes are coming; leave the sealed state // and resume compactions so L0 keeps draining (see seal). e.unseal() + // Rebinding admits mutations that sealed manifest row counts no longer + // witness; strip them so an unpublished rebound store stays fail-closed + // for replay (CO-014). Reseal recounts. Must follow unseal — the clear + // uses the normal write path. Read-only engines skip it: they admit no + // mutations, so sealed counts remain valid witnesses (and the write + // would be illegal anyway). + if e.opts.readOnly { + return nil + } + if err := e.clearSourceCacheRowCounts(); err != nil { + return err + } return nil } @@ -453,6 +536,7 @@ func (e *Engine) MarkFreshSync(syncID string) error { e.currentSync = idBytes e.freshSync = true e.freshGrantsEmpty = true + e.freshEntitlementsEmpty = true e.freshResourcesEmpty = true e.currentSyncMu.Unlock() // A fresh sync writes heavily; leave the sealed state and resume @@ -470,6 +554,7 @@ func (e *Engine) clearCurrentSync() { e.currentSync = nil e.freshSync = false e.freshGrantsEmpty = false + e.freshEntitlementsEmpty = false e.freshResourcesEmpty = false e.currentSyncMu.Unlock() } @@ -515,6 +600,16 @@ func (e *Engine) takeFreshResourcesEmpty() bool { return true } +func (e *Engine) takeFreshEntitlementsEmpty() bool { + e.currentSyncMu.Lock() + defer e.currentSyncMu.Unlock() + if !e.freshEntitlementsEmpty { + return false + } + e.freshEntitlementsEmpty = false + return true +} + // EndFreshSync clears the fresh-sync flag and flushes the memtable // + fsyncs the WAL so the data written during the sync is on disk // before the caller returns. Called by Adapter.EndSync. @@ -639,9 +734,9 @@ func (e *Engine) withWrite(fn func() error) error { // withWriteAllowSealed is withWrite without the sealed check. Reserved for // writes that are part of the sealed lifecycle itself: sync-run metadata -// stamps on a finished sync (ended_at overrides, diff links, supports_diff) -// and ResetForNewSync's wipe on the way into a new sync. Record-data writes -// must use withWrite. +// stamps on a finished sync (ended_at overrides, supports_diff), +// compactor source-cache invalidation, and ResetForNewSync's wipe on the way +// into a new sync. Record-data writes must use withWrite. func (e *Engine) withWriteAllowSealed(fn func() error) error { if err := e.checkWritableAllowSealed(); err != nil { return err diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlement_graph_sidecar.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlement_graph_sidecar.go new file mode 100644 index 00000000..75ad75b7 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlement_graph_sidecar.go @@ -0,0 +1,88 @@ +package pebble + +import ( + "context" + "errors" + + "github.com/cockroachdb/pebble/v2" + + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" +) + +// Entitlement-graph sidecar: an opaque blob (owned by pkg/sync/expand) +// holding the sync's expansion graph, so it rides the c1z instead of +// bloating the sync token. Same single-fixed-key shape as the stats +// sidecar; absent on files written by a pre-sidecar SDK. + +// encodeEntitlementGraphKey returns the engine-meta key for the single +// sync's graph blob. One sync per file, so no sync_id in the key. +func encodeEntitlementGraphKey() []byte { + buf := make([]byte, 0, 6+len("entitlement-graph")) + buf = append(buf, versionV3, typeEngineMeta) + buf = codec.AppendTupleString(buf, "entitlement-graph") + buf = codec.AppendTupleSeparator(buf) + return buf +} + +// EntitlementGraphSidecarLowerBound / UpperBound expose the sidecar's +// single-key range for cleanup and compaction. +func EntitlementGraphSidecarLowerBound() []byte { + return encodeEntitlementGraphKey() +} + +func EntitlementGraphSidecarUpperBound() []byte { + return upperBoundOf(EntitlementGraphSidecarLowerBound()) +} + +// PutEntitlementGraphSidecar stores the opaque graph blob. Same write +// barrier as the stats sidecar: callers span EndSync's sealed window. +func (e *Engine) PutEntitlementGraphSidecar(ctx context.Context, data []byte) error { + if err := ctx.Err(); err != nil { + return err + } + return e.withWriteAllowSealed(func() error { + // Re-check after waiting for the engine's write lock. The context may + // have been canceled while another writer held the lock. + if err := ctx.Err(); err != nil { + return err + } + return e.db.MetaSet(encodeEntitlementGraphKey(), data, pebble.Sync) + }) +} + +// GetEntitlementGraphSidecar returns the stored blob, or (nil, nil) if +// none exists. +func (e *Engine) GetEntitlementGraphSidecar(ctx context.Context) ([]byte, error) { + if err := ctx.Err(); err != nil { + return nil, err + } + val, closer, err := e.db.Get(encodeEntitlementGraphKey()) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return nil, nil + } + return nil, err + } + defer closer.Close() + out := make([]byte, len(val)) + copy(out, val) + if err := ctx.Err(); err != nil { + return nil, err + } + return out, nil +} + +// DeleteEntitlementGraphSidecar removes the blob (no-op when absent). +func (e *Engine) DeleteEntitlementGraphSidecar(ctx context.Context) error { + if err := ctx.Err(); err != nil { + return err + } + return e.withWriteAllowSealed(func() error { + // Re-check after waiting for the engine's write lock. The context may + // have been canceled while another writer held the lock. + if err := ctx.Err(); err != nil { + return err + } + return e.db.MetaDelete(encodeEntitlementGraphKey(), pebble.Sync) + }) +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go index e9e728e2..27074b6d 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/entitlements.go @@ -20,9 +20,24 @@ func (e *Engine) PutEntitlementRecord(ctx context.Context, r *v3.EntitlementReco // PutEntitlementRecords writes N entitlements by structured primary key. // The identity key is a pure function of the record (it contains the raw -// external id), so overwrites are idempotent and no read-before-write or -// index cleanup is needed; a within-call dedup pre-pass keeps last-wins -// semantics for same-identity duplicates in one batch. +// external id), so overwrites are idempotent; a within-call dedup +// pre-pass keeps last-wins semantics for same-identity duplicates in one +// batch. +// +// Read-before-write exists ONLY for the by_source_scope obligation: an +// overwrite that changes a row's scope stamp must clean the prior +// entry, and entitlement scope entries are value-derived. The Get is +// therefore skipped while rawdb's sourceScopeMayExist gate is unarmed: +// the unarmed gate certifies no index entry exists to clean (the only +// thing the prior value is fetched for), so an ordinary unscoped sync +// pays no per-row read at all — the pre-scope write cost, exactly. A +// stamped record still gets its index entry (stageSourceScopeChange +// always scans the NEW value and arms the gate AT STAGING, before the +// batch commits), so later records in the same call — and every call +// after — take the Get path. Rows staged Get-free earlier in the +// arming call are sound: the gate was unarmed when they staged, so no +// committed index entry existed for their identities (db.Get cannot +// see in-batch writes either way). func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.EntitlementRecord) error { if len(records) == 0 { return nil @@ -35,6 +50,7 @@ func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.Entit defer priBatch.Close() fresh := e.IsFreshSync() + skipGet := e.takeFreshEntitlementsEmpty() type dedupKey struct { id entitlementIdentity @@ -72,7 +88,23 @@ func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.Entit if err != nil { return err } - if err := priBatch.StageEntitlementPut(key, val); err != nil { + if skipGet || !e.db.SourceScopeMayExist() { + if err := priBatch.StageEntitlementPut(key, val, nil); err != nil { + return err + } + continue + } + oldVal, closer, getErr := e.db.Get(key) + switch { + case getErr == nil: + err = priBatch.StageEntitlementPut(key, val, oldVal) + closer.Close() + case errors.Is(getErr, pebble.ErrNotFound): + err = priBatch.StageEntitlementPut(key, val, nil) + default: + return fmt.Errorf("PutEntitlementRecords: get old: %w", getErr) + } + if err != nil { return err } } @@ -120,11 +152,20 @@ func (e *Engine) DeleteEntitlementRecord(ctx context.Context, externalID string) return err } key := encodeEntitlementIdentityKey(id) + oldVal, closer, getErr := e.db.Get(key) + if errors.Is(getErr, pebble.ErrNotFound) { + return nil + } + if getErr != nil { + return getErr + } batch := e.db.NewRecordBatch() defer batch.Close() - if err := batch.StageEntitlementDelete(key); err != nil { + if err := batch.StageEntitlementDelete(key, oldVal); err != nil { + closer.Close() return err } + closer.Close() if err := batch.Commit(writeOpts(e.opts.durability)); err != nil { return err } @@ -146,9 +187,20 @@ func (e *Engine) DeleteEntitlementRecordByIdentity( key := encodeEntitlementIdentityKey( entitlementIdentityFromParts(resourceTypeID, resourceID, externalID), ) + // A missing row stages nothing, matching the bare-ID delete above and the + // grant path's rule: staging unconditionally would emit index and digest + // obligations for identities that never existed. + oldVal, closer, err := e.db.Get(key) + if errors.Is(err, pebble.ErrNotFound) { + return nil + } + if err != nil { + return err + } + defer closer.Close() batch := e.db.NewRecordBatch() defer batch.Close() - if err := batch.StageEntitlementDelete(key); err != nil { + if err := batch.StageEntitlementDelete(key, oldVal); err != nil { return err } if err := batch.Commit(writeOpts(e.opts.durability)); err != nil { @@ -159,6 +211,70 @@ func (e *Engine) DeleteEntitlementRecordByIdentity( }) } +// DeleteEntitlementRecords validates every public id before staging any +// tombstone, then commits the resolved deletes in bounded chunks (deletion +// is idempotent; a mid-way error retries convergently). Missing ids are +// no-ops; an ambiguous id rejects the entire request. actingScope is the +// scope on whose behalf the tombstones act: deleting that scope's own rows +// stages no poison, while deleting a row stamped with any OTHER scope +// poisons it (CO-015); "" acts unscoped and poisons any stamped delete. +func (e *Engine) DeleteEntitlementRecords(ctx context.Context, externalIDs []string, actingScope string) error { + return e.withWrite(func() error { + identities := make([]entitlementIdentity, 0, len(externalIDs)) + seen := make(map[entitlementIdentity]struct{}, len(externalIDs)) + for _, externalID := range externalIDs { + if err := ctx.Err(); err != nil { + return err + } + id, err := e.resolveEntitlementIdentityByExternalID(ctx, externalID) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + continue + } + return err + } + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + identities = append(identities, id) + } + if len(identities) == 0 { + return nil + } + + deletes := newSourceCacheDeleteBatch(e, "entitlements-canonical", actingScope, writeOpts(e.opts.durability)) + defer deletes.close() + // The bare-id lookup map must observe every chunk AS it lands, + // not on function exit: entitlementIdentitiesForExternalID takes + // only entIDLookupMu, so a concurrent lookup between a mid-loop + // chunk commit and this function's return would serve a cached + // map listing rows already deleted on disk. The per-commit hook + // (an atomic add) closes that window and covers the + // error-after-intermediate-commit case for free. + deletes.onCommit = e.noteEntitlementKeyspaceWrite + for _, id := range identities { + key := encodeEntitlementIdentityKey(id) + oldVal, closer, err := e.db.Get(key) + if errors.Is(err, pebble.ErrNotFound) { + continue + } + if err != nil { + return err + } + if err := deletes.batch.StageEntitlementDelete(key, oldVal); err != nil { + _ = closer.Close() + return err + } + _ = closer.Close() + if err := deletes.staged(true); err != nil { + return err + } + } + return deletes.commit(true) + }) +} + func (e *Engine) IterateEntitlements(ctx context.Context, yield func(*v3.EntitlementRecord) bool) error { prefix := encodeEntitlementPrefix() iter, err := e.db.NewIter(&pebble.IterOptions{ diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest.go index b173e5de..c13e4eb5 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grant_digest.go @@ -12,6 +12,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb" ) @@ -102,16 +103,89 @@ func grantPrincipalBucketHash64(encodedPrincipalSegments []byte) uint64 { return xxhash.Sum64(encodedPrincipalSegments) } +// DigestBucketHashBits is how many of PrincipalBucketHash's leading bits +// actually select a digest bucket: the stored bucket hash is truncated to +// this width, so bucket levels beyond it cannot subdivide further and are +// rejected (see PrincipalDigestBucket, GetEntitlementGrantDigestNodes, +// ScanEntitlementGrantBucket). +// +// ABI: the stored truncation width, pinned to GrantDigestABIVersion. It may +// only grow, and only under an index-migration bump — which is why it is a +// named constant rather than a literal in PrincipalBucketHash's signature: +// widening the addressable bucket space must not change that signature. +const DigestBucketHashBits = digestBucketHashLen * 8 + +// PrincipalBucketHash is the public form of the grant digest's bucket +// address for a principal: the full 64-bit xxHash64 of the principal's +// ENCODED identity segments (see grantPrincipalBucketHash64). Identity +// only — never the principal's attributes — so a principal keeps its +// bucket across syncs. +// +// Use PrincipalDigestBucket to turn this into a bucket at a given level; +// it owns the index math below so callers never hand-derive it: +// +// bucket, _ := PrincipalDigestBucket(rt, id, level) +// nodes, _, _ := r.GetEntitlementGrantDigestNodes(ctx, ent, level) +// _ = r.ScanEntitlementGrantBucket(ctx, ent, bucket, yield) +// +// Cost: levels at or below the digest's native level +// (GetEntitlementGrantDigest().Level) fold the stored leaves — one cheap +// contiguous scan. A finer level is exact but costs a full scan of the +// entitlement's grant index on every call, so prefer the native level +// unless narrowing a bucket is worth that. +// +// Contract: the bucket at level L holds exactly the principals whose top +// L bits of this hash equal the bucket index — the same index +// GetEntitlementGrantDigestNodes(L) reports and ScanEntitlementGrantBucket +// takes. Only the leading DigestBucketHashBits bits are stored, so a +// level past that has no addressable bucket: PrincipalDigestBucket and +// the read APIs all ERROR on such a level rather than silently folding it +// to DigestBucketHashBits, so a caller's precomputed placement and what +// the engine actually scans never quietly diverge. L == 0 is the whole +// entitlement (index 0). +// +// ABI: pinned to GrantDigestABIVersion alongside GrantContentHash. Two +// SDK builds must place the same principal in the same bucket, so the +// input framing changes only under an index-migration bump. +func PrincipalBucketHash(principalRT, principalID string) uint64 { + enc := codec.AppendTupleStrings(make([]byte, 0, 64), principalRT, principalID) + return grantPrincipalBucketHash64(enc) +} + +// PrincipalDigestBucket places a principal into its grant-digest bucket +// at level: the connectorstore.GrantDigestBucket a caller outside this +// package would otherwise have to hand-derive from PrincipalBucketHash's +// raw shift formula. Index is the top level bits of PrincipalBucketHash, +// matching exactly what GetEntitlementGrantDigestNodes(level) reports and +// ScanEntitlementGrantBucket(level, Index) scans. +// +// level must be in [0, DigestBucketHashBits] — 0 is the whole entitlement +// (Index always 0); past DigestBucketHashBits there is no finer +// addressable bucket, and this errors rather than silently returning an +// Index computed at a resolution the stored hash doesn't have. The read +// APIs enforce the same bound, so a bucket built here is always valid to +// pass to them. +func PrincipalDigestBucket(principalRT, principalID string, level int) (connectorstore.GrantDigestBucket, error) { + if level < 0 || level > DigestBucketHashBits { + return connectorstore.GrantDigestBucket{}, fmt.Errorf("pebble: grant-digest level %d out of range [0, %d]", level, DigestBucketHashBits) + } + if level == 0 { + return connectorstore.GrantDigestBucket{Level: 0, Index: 0}, nil + } + idx := uint32(PrincipalBucketHash(principalRT, principalID) >> (64 - level)) //nolint:gosec // level <= DigestBucketHashBits (16), so the shift leaves at most 16 bits + return connectorstore.GrantDigestBucket{Level: level, Index: idx}, nil +} + // principalBucketHash is the from-identity form of the bucket hash: // the stored digestBucketHashLen key bytes for a principal given its -// decoded identity. Encodes the segments exactly as the primary grant -// key does, then hashes — so it MUST agree with hashing the spliced -// key region (pinned by TestGrantDigestSpliceMatchesEncode). Returns a +// decoded identity — the truncation of PrincipalBucketHash that index +// keys carry. Encodes the segments exactly as the primary grant key +// does, then hashes — so it MUST agree with hashing the spliced key +// region (pinned by TestGrantDigestSpliceMatchesEncode). Returns a // fresh slice. func principalBucketHash(principalRT, principalID string) []byte { - enc := codec.AppendTupleStrings(make([]byte, 0, 64), principalRT, principalID) var full [8]byte - binary.BigEndian.PutUint64(full[:], grantPrincipalBucketHash64(enc)) + binary.BigEndian.PutUint64(full[:], PrincipalBucketHash(principalRT, principalID)) out := make([]byte, digestBucketHashLen) copy(out, full[:]) return out diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go index 8ec2a48f..a4afeab7 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/grants.go @@ -130,25 +130,28 @@ func (e *Engine) PutGrantRecords(ctx context.Context, records ...*v3.GrantRecord if err != nil { return err } - hadOld := false - if !skipGet { - _, closer, getErr := e.db.Get(key) - switch { - case getErr == nil: - hadOld = true - closer.Close() - case errors.Is(getErr, pebble.ErrNotFound): - // no prior record — write unconditionally - default: - return fmt.Errorf("PutGrantRecords: get old: %w", getErr) - } - } // One typed op stages the row and everything it owes: // prior-row index cleanup, by_principal, needs_expansion, // digest invalidation. Index keys derive from the primary - // key (identity-encoded), so the prior VALUE is not needed - // for cleanup — the Get above reduces to an existence probe. - if err := batch.StageGrantPutInline(key, val, hadOld, r.GetNeedsExpansion()); err != nil { + // key (identity-encoded); the prior value is retained only + // long enough to clean a changed source-scope index entry. + if skipGet { + if err := batch.StageGrantPutInline(key, val, nil, r.GetNeedsExpansion()); err != nil { + return err + } + continue + } + oldVal, closer, getErr := e.db.Get(key) + switch { + case getErr == nil: + err = batch.StageGrantPutInline(key, val, oldVal, r.GetNeedsExpansion()) + closer.Close() + case errors.Is(getErr, pebble.ErrNotFound): + err = batch.StageGrantPutInline(key, val, nil, r.GetNeedsExpansion()) + default: + return fmt.Errorf("PutGrantRecords: get old: %w", getErr) + } + if err != nil { return err } } @@ -244,46 +247,50 @@ func (e *Engine) PutExpandedGrantRecords(ctx context.Context, records []*v3.Gran if dedup != nil && dedup[id] != i { continue } - ext := r.GetExternalId() - keyScratch = appendGrantIdentityKey(keyScratch[:0], id) + if err := func() error { + ext := r.GetExternalId() + keyScratch = appendGrantIdentityKey(keyScratch[:0], id) - hadOld := false - oldVal, closer, getErr := e.db.Get(keyScratch) - switch { - case getErr == nil: - // Preserve the prior record's expansion side-state + - // discovered_at (StoreExpandedGrants contract). The prior - // value's index cleanup is the typed op's obligation. - hadOld = true - prior.Reset() - if err := unmarshalRecord(oldVal, &prior); err != nil { - closer.Close() - return fmt.Errorf("PutExpandedGrantRecords: unmarshal prior %q: %w", ext, err) + oldVal, closer, getErr := e.db.Get(keyScratch) + switch { + case getErr == nil: + defer closer.Close() + // Preserve the prior record's expansion side-state + + // discovered_at (StoreExpandedGrants contract). The prior + // value's index cleanup is the typed op's obligation. + prior.Reset() + if err := unmarshalRecord(oldVal, &prior); err != nil { + return fmt.Errorf("PutExpandedGrantRecords: unmarshal prior %q: %w", ext, err) + } + r.SetExpansion(prior.GetExpansion()) + r.SetNeedsExpansion(prior.GetNeedsExpansion()) + r.SetDiscoveredAt(prior.GetDiscoveredAt()) + r.SetSourceScopeKey(prior.GetSourceScopeKey()) + case errors.Is(getErr, pebble.ErrNotFound): + // No prior record: discovered_at is stamped below unless the + // translation already carried one. + default: + return fmt.Errorf("PutExpandedGrantRecords: get old %q: %w", ext, getErr) + } + if r.GetDiscoveredAt() == nil { + r.SetDiscoveredAt(now) } - r.SetExpansion(prior.GetExpansion()) - r.SetNeedsExpansion(prior.GetNeedsExpansion()) - r.SetDiscoveredAt(prior.GetDiscoveredAt()) - closer.Close() - case errors.Is(getErr, pebble.ErrNotFound): - // No prior record: discovered_at is stamped below unless the - // translation already carried one. - default: - return fmt.Errorf("PutExpandedGrantRecords: get old %q: %w", ext, getErr) - } - if r.GetDiscoveredAt() == nil { - r.SetDiscoveredAt(now) - } - val, err := marshalRecordAppend(valScratch[:0], r) - if err != nil { - return err - } - valScratch = val - // Deferred regime: arms the rebuild marker, cleans the prior - // needs_expansion entry (by_principal is excised+rebuilt at - // seal), stages row + conditional needs_expansion + digest - // invalidation. - if err := batch.StageGrantPutDeferred(keyScratch, val, hadOld, r.GetNeedsExpansion()); err != nil { + val, err := marshalRecordAppend(valScratch[:0], r) + if err != nil { + return err + } + valScratch = val + // Deferred regime: arms the rebuild marker, cleans the prior + // needs_expansion entry (by_principal is excised+rebuilt at + // seal), stages row + conditional needs_expansion + digest + // invalidation. + var priorVal []byte + if getErr == nil { + priorVal = oldVal + } + return batch.StageGrantPutDeferred(keyScratch, val, priorVal, r.GetNeedsExpansion()) + }(); err != nil { return err } } @@ -339,7 +346,7 @@ func (e *Engine) PutSynthesizedGrantRecords(ctx context.Context, records []*v3.G valScratch = val // Deferred regime; the caller guarantees brand-new identities, // so there is no prior row to clean (hadOldVal=false). - if err := batch.StageGrantPutDeferred(keyScratch, val, false, r.GetNeedsExpansion()); err != nil { + if err := batch.StageGrantPutDeferred(keyScratch, val, nil, r.GetNeedsExpansion()); err != nil { return err } } @@ -742,7 +749,7 @@ func (e *Engine) putSynthesizedGrantContributionsBatch(ctx context.Context, reco valScratch = val // Deferred regime: synthesized grants are brand-new (no prior // row) and never expandable (needsExpansion=false). - if err := batch.StageGrantPutDeferred(keyScratch, val, false, false); err != nil { + if err := batch.StageGrantPutDeferred(keyScratch, val, nil, false); err != nil { return err } } @@ -876,7 +883,7 @@ func (e *Engine) UnsafePutUniqueGrantRecords(ctx context.Context, records ...*v3 // the typed op splices index keys from the primary key on the // staging goroutine — cheaper than the encode-from-identity // the parallel workers used to do, and unforgettable. - if err := batch.StageGrantPutInline(enc[i].priKey, enc[i].priVal, false, enc[i].needsExpansion); err != nil { + if err := batch.StageGrantPutInline(enc[i].priKey, enc[i].priVal, nil, enc[i].needsExpansion); err != nil { return err } } @@ -1071,20 +1078,21 @@ func (e *Engine) deleteGrantsByIdentityChunkLocked(ids []grantIdentity) error { func (e *Engine) stageGrantDeleteIfPresentLocked(batch *rawdb.RecordBatch, id grantIdentity) (bool, error) { key := encodeGrantIdentityKey(id) - // Existence probe only: delete of non-existent stays a no-op, and - // the typed op derives all cleanup keys from the primary key. - _, closer, err := e.db.Get(key) + oldVal, closer, err := e.db.Get(key) if err != nil { if errors.Is(err, pebble.ErrNotFound) { return false, nil } return false, err } - closer.Close() - - if err := batch.StageGrantDelete(key); err != nil { + // StageGrantDelete derives the source-scope index cleanup from oldVal, so + // the closer stays open across the call. It copies what it keeps, so the + // staged batch holds nothing that outlives the release below. + if err := batch.StageGrantDelete(key, oldVal); err != nil { + closer.Close() return false, err } + closer.Close() return true, nil } @@ -1113,11 +1121,14 @@ func grantIndexKeys(r *v3.GrantRecord) [][]byte { if err != nil { return nil } - keys := make([][]byte, 0, 2) + keys := make([][]byte, 0, 3) keys = append(keys, encodeGrantByPrincipalIdentityIndexKey(id)) if r.GetNeedsExpansion() { keys = append(keys, encodeGrantByNeedsExpansionIdentityIndexKey(id)) } + if scope := r.GetSourceScopeKey(); scope != "" { + keys = append(keys, encodeGrantBySourceScopeIndexKey(scope, id)) + } return keys } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go deleted file mode 100644 index bf7902bf..00000000 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/if_newer.go +++ /dev/null @@ -1,293 +0,0 @@ -package pebble - -import ( - "context" - "errors" - "fmt" - - "github.com/cockroachdb/pebble/v2" - "google.golang.org/protobuf/types/known/timestamppb" - - v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" -) - -// *IfNewer upsert methods. Mirror the SQLite engine's -// PutGrantsIfNewer / PutResourcesIfNewer / PutEntitlementsIfNewer / -// PutResourceTypesIfNewer semantics: only overwrite the existing -// record when the incoming record's discovered_at is strictly newer. -// -// Used by partial-sync workflows (SyncTypePartialUpserts / -// SyncTypePartialDeletions) where a connector replays a recent -// window of changes and must not regress an existing record's -// discovered_at to an older timestamp. -// -// Mechanism: for each candidate record we read the existing record -// (if any), compare discovered_at, and decide. Records that pass the -// freshness check go into a single batch and commit once. The -// fresh-sync write path is disabled here — *IfNewer is by definition -// not a fresh sync (we're filtering against existing data). - -// PutGrantRecordsIfNewer writes records that are strictly newer than -// the stored copy. Records without a discovered_at are treated as -// "always write" (caller is asserting freshness explicitly). -func (e *Engine) PutGrantRecordsIfNewer(ctx context.Context, records ...*v3.GrantRecord) error { - if len(records) == 0 { - return nil - } - return e.withWrite(func() error { - if err := e.requireCurrentSync(); err != nil { - return err - } - batch := e.db.NewRecordBatch() - defer batch.Close() - // No inline hash-index or digest maintenance here: both are - // derived at seal time (the fused deferred pass). But IfNewer is - // the partial-sync path — it mutates a CLONED sealed file whose - // digests are built — so StageGrantPutInline's derivers stage - // the touched entitlements' digest invalidation whenever - // digests are present (StageGrantDigestInvalidation deriver, - // records.go). - written := 0 - for _, r := range records { - if r == nil { - continue - } - id, err := grantIdentityFromRecord(r) - if err != nil { - return err - } - key := encodeGrantIdentityKey(id) - hadOld := false - oldVal, closer, getErr := e.db.Get(key) - switch { - case getErr == nil: - write, err := discoveredAtIsNewerThanRaw(r.GetDiscoveredAt(), oldVal, grantDiscoveredAtField) - if err != nil { - closer.Close() - return fmt.Errorf("PutGrantRecordsIfNewer: scan old discovered_at: %w", err) - } - if !write { - closer.Close() - continue - } - hadOld = true - closer.Close() - case errors.Is(getErr, pebble.ErrNotFound): - // no existing record — write unconditionally - default: - return fmt.Errorf("PutGrantRecordsIfNewer: get: %w", getErr) - } - val, err := marshalRecord(r) - if err != nil { - return err - } - // Inline regime: the typed op stages the row plus prior-row - // index cleanup, both index entries, and digest invalidation - // (this IS the partial-sync path the invalidation exists for). - if err := batch.StageGrantPutInline(key, val, hadOld, r.GetNeedsExpansion()); err != nil { - return err - } - written++ - } - if written == 0 { - return nil - } - return batch.Commit(writeOpts(e.opts.durability)) - }) -} - -// PutResourceRecordsIfNewer writes resources only when the incoming -// discovered_at is strictly newer than the stored copy. -func (e *Engine) PutResourceRecordsIfNewer(ctx context.Context, records ...*v3.ResourceRecord) error { - if len(records) == 0 { - return nil - } - return e.withWrite(func() error { - if err := e.requireCurrentSync(); err != nil { - return err - } - batch := e.db.NewRecordBatch() - defer batch.Close() - written := 0 - for _, r := range records { - if r == nil { - continue - } - key := encodeResourceKey(r.GetResourceTypeId(), r.GetResourceId()) - oldVal, closer, getErr := e.db.Get(key) - switch { - case getErr == nil: - write, err := discoveredAtIsNewerThanRaw(r.GetDiscoveredAt(), oldVal, resourceDiscoveredAtField) - if err != nil { - closer.Close() - return fmt.Errorf("PutResourceRecordsIfNewer: scan old discovered_at: %w", err) - } - if !write { - closer.Close() - continue - } - val, err := marshalRecord(r) - if err != nil { - closer.Close() - return err - } - // Typed op consumes the prior value for by_parent cleanup. - err = batch.StageResourcePut(key, val, oldVal, r.GetResourceTypeId(), r.GetResourceId()) - closer.Close() - if err != nil { - return err - } - written++ - continue - case errors.Is(getErr, pebble.ErrNotFound): - default: - return fmt.Errorf("PutResourceRecordsIfNewer: get: %w", getErr) - } - val, err := marshalRecord(r) - if err != nil { - return err - } - if err := batch.StageResourcePut(key, val, nil, r.GetResourceTypeId(), r.GetResourceId()); err != nil { - return err - } - written++ - } - if written == 0 { - return nil - } - return batch.Commit(writeOpts(e.opts.durability)) - }) -} - -// PutEntitlementRecordsIfNewer writes entitlements only when newer. -func (e *Engine) PutEntitlementRecordsIfNewer(ctx context.Context, records ...*v3.EntitlementRecord) error { - if len(records) == 0 { - return nil - } - return e.withWrite(func() error { - if err := e.requireCurrentSync(); err != nil { - return err - } - batch := e.db.NewRecordBatch() - defer batch.Close() - written := 0 - for _, r := range records { - if r == nil { - continue - } - id, err := entitlementIdentityFromRecord(r) - if err != nil { - return err - } - key := encodeEntitlementIdentityKey(id) - oldVal, closer, getErr := e.db.Get(key) - switch { - case getErr == nil: - write, err := discoveredAtIsNewerThanRaw(r.GetDiscoveredAt(), oldVal, entitlementDiscoveredAtField) - if err != nil { - closer.Close() - return fmt.Errorf("PutEntitlementRecordsIfNewer: scan old discovered_at: %w", err) - } - if !write { - closer.Close() - continue - } - closer.Close() - case errors.Is(getErr, pebble.ErrNotFound): - default: - return fmt.Errorf("PutEntitlementRecordsIfNewer: get: %w", getErr) - } - val, err := marshalRecord(r) - if err != nil { - return err - } - if err := batch.StageEntitlementPut(key, val); err != nil { - return err - } - written++ - } - if written == 0 { - return nil - } - if err := batch.Commit(writeOpts(e.opts.durability)); err != nil { - return err - } - e.noteEntitlementKeyspaceWrite() - return nil - }) -} - -// PutResourceTypeRecordsIfNewer writes resource_types only when newer. -func (e *Engine) PutResourceTypeRecordsIfNewer(ctx context.Context, records ...*v3.ResourceTypeRecord) error { - if len(records) == 0 { - return nil - } - return e.withWrite(func() error { - if err := e.requireCurrentSync(); err != nil { - return err - } - batch := e.db.NewRecordBatch() - defer batch.Close() - written := 0 - for _, r := range records { - if r == nil { - continue - } - key := encodeResourceTypeKey(r.GetExternalId()) - oldVal, closer, getErr := e.db.Get(key) - switch { - case getErr == nil: - write, err := discoveredAtIsNewerThanRaw(r.GetDiscoveredAt(), oldVal, resourceTypeDiscoveredAtField) - if err != nil { - closer.Close() - return fmt.Errorf("PutResourceTypeRecordsIfNewer: scan old discovered_at: %w", err) - } - closer.Close() - if !write { - continue - } - case errors.Is(getErr, pebble.ErrNotFound): - default: - return fmt.Errorf("PutResourceTypeRecordsIfNewer: get: %w", getErr) - } - val, err := marshalRecord(r) - if err != nil { - return err - } - if err := batch.StageResourceTypePut(key, val); err != nil { - return err - } - written++ - } - if written == 0 { - return nil - } - return batch.Commit(writeOpts(e.opts.durability)) - }) -} - -// discoveredAtIsNewer returns true iff incoming is strictly after -// existing. Matches SQLite's `EXCLUDED.discovered_at > X.discovered_at` -// semantics, including the NULL-propagation rules: -// -// - nil incoming → false (SQLite `NULL > X` is NULL, i.e. don't -// write). Adapter-level PutXxxIfNewer methods stamp DiscoveredAt -// to time.Now() before calling here, so production code never -// hits this branch; direct engine callers must supply a non-nil -// DiscoveredAt to mean "write this". -// - nil existing → true (no prior record at this key, so the -// incoming row wins by default — SQLite's INSERT-on-conflict -// reduces to a plain INSERT). -// - both non-nil → strict After comparison. -// -// Keep this in sync with extractAndStripExpansion / putGrantsInternal -// in pkg/dotc1z/grants.go if the SQLite IfNewer path ever changes. -func discoveredAtIsNewer(incoming, existing *timestamppb.Timestamp) bool { - if incoming == nil { - return false - } - if existing == nil { - return true - } - return incoming.AsTime().After(existing.AsTime()) -} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/index_migrations.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/index_migrations.go index b94e8a67..3f8bc812 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/index_migrations.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/index_migrations.go @@ -63,6 +63,14 @@ type indexMigration struct { // the corresponding index for any existing c1z that doesn't have // it yet. // +// OBLIGATION: migrations run after the Open-time presence probes +// (grant digests, source-scope gate). A migration that creates +// by_source_scope entries must re-probe or arm the gate +// (SetSourceScopeMayExist / ProbeSourceScopeMayExist) before +// returning — leaving entries behind an unarmed gate makes later +// overwrites/deletes skip index cleanup (false-with-entries, the one +// unsound gate state). +// // Deliberately empty today. The by_entitlement_principal_hash index + // grant digests are NOT backfilled at Open: they are rebuilt from the // primaries at every seal (the fused deferred pass / BuildGrantDigests), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/families.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/families.go index 72a104c7..d559898e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/families.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/families.go @@ -17,6 +17,7 @@ package rawdb import ( "context" + "sync/atomic" "github.com/cockroachdb/pebble/v2" ) @@ -33,8 +34,14 @@ type Stager interface { } // batch is the shared staged-write core the family types embed. +// open points at the owning DB's per-family outstanding-batch counter +// (batchAccounting): minting increments it, the first Close decrements +// it, and DB.Close reports any nonzero balance as a leak. Close also +// nils the pebble handle so a second Close is a no-op instead of a +// double release of a pooled object (§5.2 ownership transfer). type batch struct { - b *pebble.Batch + b *pebble.Batch + open *atomic.Int64 } // Set stages key → val. @@ -58,15 +65,27 @@ func (b *batch) Len() int { return len(b.b.Repr()) } // Commit applies the staged writes with the given write options. func (b *batch) Commit(o *pebble.WriteOptions) error { return b.b.Commit(o) } -// Close releases the batch. Safe after Commit. -func (b *batch) Close() error { return b.b.Close() } +// Close releases the batch. Safe after Commit, and idempotent: the +// first call transfers ownership back to pebble's pool and nils the +// handle; later calls return nil without touching the pooled object. +func (b *batch) Close() error { + if b.b == nil { + return nil + } + err := b.b.Close() + b.b = nil + if b.open != nil { + b.open.Add(-1) + } + return err +} // === record family: grants / entitlements / resources / resource types === // // The primary record keyspaces plus their inline-maintained index // families and the digest-invalidation markers a record mutation owes. // Clients: the Put*Records paths, the expanded/synthesized grant -// writers, the IfNewer partial-sync paths, and delete paths. +// writers, and delete paths. // // RecordBatch exposes NO generic staging. The only way to stage a // record mutation is a typed Stage* operation (records.go) that @@ -82,15 +101,45 @@ type RecordBatch struct { core batch db *DB scratch []byte + // actingSourceScope is the scope on whose behalf this batch's + // deletes act (SetActingSourceScope); "" means unscoped. Consulted + // by stageSourceScopeCleanup's poison decision (CO-015). + actingSourceScope string + // poisonStaged / poisonEvents: per-batch dedup set and ordered log + // of staged poison markers; events are delivered to the DB's poison + // observer after a successful Commit. Nil until the first poison — + // the ordinary mutation path pays nothing. + poisonStaged map[PoisonEvent]struct{} + poisonEvents []PoisonEvent } // NewRecordBatch mints a batch for record-keyspace mutations. func (d *DB) NewRecordBatch() *RecordBatch { - return &RecordBatch{core: batch{b: d.newBatch()}, db: d} + d.acct.record.Add(1) + return &RecordBatch{core: batch{b: d.newBatch(), open: &d.acct.record}, db: d} } -// Commit applies the staged writes with the given write options. -func (rb *RecordBatch) Commit(o *pebble.WriteOptions) error { return rb.core.Commit(o) } +// Commit applies the staged writes with the given write options. Poison +// events staged in this batch are delivered to the observer only after +// the commit lands — a failed commit stages nothing durable and logs +// nothing. +func (rb *RecordBatch) Commit(o *pebble.WriteOptions) error { + if rb.db.testRecordCommitHook != nil { + if err := rb.db.testRecordCommitHook(); err != nil { + return err + } + } + if err := rb.core.Commit(o); err != nil { + return err + } + if rb.db.poisonObserver != nil { + for _, ev := range rb.poisonEvents { + rb.db.poisonObserver(ev) + } + } + rb.poisonEvents = rb.poisonEvents[:0] + return nil +} // Close releases the batch. Safe after Commit. func (rb *RecordBatch) Close() error { return rb.core.Close() } @@ -122,7 +171,10 @@ type SessionBatch struct { } // NewSessionBatch mints a batch for session-keyspace mutations. -func (d *DB) NewSessionBatch() *SessionBatch { return &SessionBatch{batch{b: d.newBatch()}} } +func (d *DB) NewSessionBatch() *SessionBatch { + d.acct.session.Add(1) + return &SessionBatch{batch{b: d.newBatch(), open: &d.acct.session}} +} // === engine-meta family === // @@ -137,6 +189,48 @@ func (d *DB) MetaSet(key, val []byte, o *pebble.WriteOptions) error { return d.s // MetaDelete removes one engine-meta / fixed-key row. func (d *DB) MetaDelete(key []byte, o *pebble.WriteOptions) error { return d.delete(key, o) } +// === source-cache manifest family === + +// SourceCacheSet writes one manifest or compatibility record. Row-copy +// mutations use RecordBatch because their scope-index obligations belong to +// the record family. +func (d *DB) SourceCacheSet(key, val []byte, o *pebble.WriteOptions) error { + return d.set(key, val, o) +} + +func (d *DB) SourceCacheDelete(key []byte, o *pebble.WriteOptions) error { + return d.delete(key, o) +} + +// SourceCacheKV is one manifest-entry write for SourceCacheSetMulti. +type SourceCacheKV struct { + Key []byte + Val []byte +} + +// SourceCacheSetMulti rewrites one PAGE of manifest entries as a single +// atomic commit. The seal and rebind count-rewrites call it once per +// bounded page (manifestRewritePageRows in the engine's source_cache.go) +// so batch size and caller memory are bounded by the page, not the +// manifest (scopes × row kinds, connector-controlled). Durability is +// the caller's argument: the rebind clear commits intermediate pages +// NoSync and syncs the final page, which persists the WAL prefix +// covering every earlier one; the seal rides NoSync entirely, hardened +// by the ended_at stamp's fsync. +func (d *DB) SourceCacheSetMulti(kvs []SourceCacheKV, o *pebble.WriteOptions) error { + if len(kvs) == 0 { + return nil + } + b := d.newBatch() + defer func() { _ = b.Close() }() + for _, kv := range kvs { + if err := b.Set(kv.Key, kv.Val, nil); err != nil { + return err + } + } + return b.Commit(o) +} + // === digest family: build / repair === // // The seal-time digest build and the post-seal repair pass are the @@ -151,7 +245,10 @@ type DigestBatch struct { } // NewDigestBatch mints a batch for digest-keyspace writes. -func (d *DB) NewDigestBatch() *DigestBatch { return &DigestBatch{batch{b: d.newBatch()}} } +func (d *DB) NewDigestBatch() *DigestBatch { + d.acct.digest.Add(1) + return &DigestBatch{batch{b: d.newBatch(), open: &d.acct.digest}} +} // DigestSet writes one digest node/root row outside a batch (the // global root stamp at the end of build/repair). @@ -166,6 +263,15 @@ func (d *DB) DropKeyRange(start, end []byte, o *pebble.WriteOptions) error { // === ingest family: deferred index build, digest build, bulk import, // synth-grant layer, id-index migration, compactor merges === +// +// OBLIGATION: SST ingest bypasses the typed record ops, so it also +// bypasses the sourceScopeMayExist arming they perform. Any caller +// whose SSTs can contain by_source_scope index entries must re-probe +// or arm the gate after a successful ingest (bulk import's Finish +// does; the synth layer and rebuild compactors provably never emit +// scope keys; fold arms at NewFoldBatch). Ingesting scope entries +// behind an unarmed gate is the one unsound gate state — later +// overwrites/deletes would skip index cleanup and orphan entries. // IngestSSTs ingests externally built SSTs (bulk import, synth-layer // segments, compactor merge output). Paths must live on DB.FS(). diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/keyspace.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/keyspace.go index 19ee7909..ab2d9226 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/keyspace.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/keyspace.go @@ -34,7 +34,11 @@ const ( TypeCounter byte = 0x08 TypeSession byte = 0x09 TypeDigest byte = 0x0A - TypeEngineMeta byte = 0xFF + // TypeSourceCache stores the per-scope replay manifest. It follows + // TypeDigest because 0x0A was assigned to digests before source-cache + // replay was extracted onto the current keyspace. + TypeSourceCache byte = 0x0B + TypeEngineMeta byte = 0xFF ) // Index-discriminator bytes (second byte after TypeIndex). One byte @@ -57,6 +61,9 @@ const ( // pass, never maintained inline. See the engine's digest.go and // grant_digest.go. IdxGrantByEntitlementPrincipalHash byte = 0x08 + IdxGrantBySourceScope byte = 0x09 + IdxEntitlementBySourceScope byte = 0x0A + IdxResourceBySourceScope byte = 0x0B ) // GrantPrimaryKeyPrefixLen is the byte length of the grant primary-key @@ -167,6 +174,125 @@ func AppendGrantByNeedsExpansionKeyFromPrimary(dst, primaryKey []byte) ([]byte, return append(dst, primaryKey[2:]...), true } +// AppendBySourceScopeKeyFromPrimary builds a by_source_scope index key +// from a primary record key. The primary tail is copied byte-for-byte, so +// replay can reconstruct the primary key without decoding the record. +func AppendBySourceScopeKeyFromPrimary(dst, primaryKey []byte, scopeKey string) ([]byte, bool) { + if scopeKey == "" || len(primaryKey) < 3 || primaryKey[0] != VersionV3 || primaryKey[2] != 0 { + return dst, false + } + var indexID byte + switch primaryKey[1] { + case TypeGrant: + if _, ok := SplitGrantPrimaryKey(primaryKey); !ok { + return dst, false + } + indexID = IdxGrantBySourceScope + case TypeEntitlement: + indexID = IdxEntitlementBySourceScope + case TypeResource: + indexID = IdxResourceBySourceScope + default: + return dst, false + } + dst = append(dst, VersionV3, TypeIndex, indexID) + dst = codec.AppendTupleSeparator(dst) + dst = codec.AppendTupleStrings(dst, scopeKey) + dst = codec.AppendTupleSeparator(dst) + return append(dst, primaryKey[3:]...), true +} + +// SourceScopeIndexPrefix bounds all rows of one record kind stamped with +// scopeKey. +func SourceScopeIndexPrefix(recordType byte, scopeKey string) ([]byte, bool) { + var indexID byte + switch recordType { + case TypeGrant: + indexID = IdxGrantBySourceScope + case TypeEntitlement: + indexID = IdxEntitlementBySourceScope + case TypeResource: + indexID = IdxResourceBySourceScope + default: + return nil, false + } + buf := []byte{VersionV3, TypeIndex, indexID} + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, scopeKey) + return codec.AppendTupleSeparator(buf), true +} + +// Sub-family discriminators inside TypeSourceCache. Manifest entries +// were laid down as type byte + tuple separator (0x00) + tuple, so the +// separator byte doubles as the entry family's discriminator — existing +// artifacts parse unchanged. Poison markers take the next byte, keeping +// the two ranges disjoint while both stay inside the one family the +// replay-state invalidation wipes. +const ( + sourceCacheEntrySubFamily byte = 0x00 + sourceCachePoisonSubFamily byte = 0x01 +) + +// SourceCacheEntryKey addresses one manifest row. +func SourceCacheEntryKey(rowKind, scopeKey string) []byte { + buf := []byte{VersionV3, TypeSourceCache} + buf = codec.AppendTupleSeparator(buf) + return codec.AppendTupleStrings(buf, rowKind, scopeKey) +} + +// SourceCacheEntryBounds bounds manifest entries ONLY — poison markers +// are deliberately outside so manifest iterators (seal counting, count +// clearing) never see non-entry values. +func SourceCacheEntryBounds() ([]byte, []byte) { + lo := []byte{VersionV3, TypeSourceCache, sourceCacheEntrySubFamily} + return lo, UpperBound(lo) +} + +// SourceCachePoisonKey addresses the per-(row_kind, scope) poison marker +// (CO-015): present means the source sync observed a row-partition +// violation against that scope — a cross-scope restamp or an +// out-of-scope delete removed a row the scope's manifest entry vouches +// for — so the scope must not be trusted as a replay source. The value +// is empty; presence is the verdict, cause goes to logs at staging time. +func SourceCachePoisonKey(rowKind, scopeKey string) []byte { + buf := []byte{VersionV3, TypeSourceCache, sourceCachePoisonSubFamily} + buf = codec.AppendTupleSeparator(buf) + return codec.AppendTupleStrings(buf, rowKind, scopeKey) +} + +// SourceCachePoisonBounds bounds all poison markers. +func SourceCachePoisonBounds() ([]byte, []byte) { + lo := []byte{VersionV3, TypeSourceCache, sourceCachePoisonSubFamily} + return lo, UpperBound(lo) +} + +// SourceCacheFamilyBounds bounds the whole source-cache family: manifest +// entries and poison markers. This is the replay-state invalidation +// range — an artifact whose validators are wiped has nothing left for +// poison to protect, so the markers go with them. +func SourceCacheFamilyBounds() ([]byte, []byte) { + lo := []byte{VersionV3, TypeSourceCache} + return lo, UpperBound(lo) +} + +// RowKindForRecordType maps a primary record type byte to the row-kind +// string used in manifest and poison keys. These strings are an on-disk +// ABI shared with pkg/sourcecache's RowKind constants; the engine's +// sourceCacheRowKindSpecs pins the same correspondence from the other +// side. +func RowKindForRecordType(recordType byte) (string, bool) { + switch recordType { + case TypeResource: + return "resources", true + case TypeEntitlement: + return "entitlements", true + case TypeGrant: + return "grants", true + default: + return "", false + } +} + // === resource index key + value scanners === // EncodeResourceByParentIndexKey: index of children-by-parent: @@ -218,6 +344,38 @@ func ScanResourceParentRaw(value []byte) (string, string, error) { return rt, id, nil } +// ScanSourceScopeKeyRaw extracts the string source_scope_key field from a +// marshaled record. Last occurrence wins, matching protobuf semantics for +// scalar fields. +func ScanSourceScopeKeyRaw(value []byte, field protowire.Number) (string, error) { + var scopeKey string + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return "", protowire.ParseError(n) + } + value = value[n:] + if num != field { + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return "", protowire.ParseError(n) + } + value = value[n:] + continue + } + if typ != protowire.BytesType { + return "", fmt.Errorf("raw record: source_scope_key field %d has wire type %v", field, typ) + } + raw, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", protowire.ParseError(n) + } + scopeKey = string(raw) + value = value[n:] + } + return scopeKey, nil +} + // ScanResourceRefRaw extracts (resource_type_id, resource_id) from a // marshaled ResourceRef/ResourceId-shaped message (fields 1 and 2). func ScanResourceRefRaw(value []byte) (string, string, error) { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/rawdb.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/rawdb.go index 9b791235..b46db6cc 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/rawdb.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/rawdb.go @@ -31,6 +31,7 @@ package rawdb import ( "context" "errors" + "fmt" "io" "sync/atomic" "testing" @@ -68,12 +69,76 @@ type DB struct { // build, cleared by the engine's drop/reset paths. grantDigestsPresent atomic.Bool + // sourceScopeMayExist gates the record ops' source-scope index + // obligations, the same shape as grantDigestsPresent: false + // certifies that NO by_source_scope index entry exists in any of + // the three families, so overwrite/delete cleanup owes nothing and + // the prior-value scope scans (and the entitlement write path's + // read-before-write, whose only purpose is feeding them) can be + // skipped — the ordinary unscoped sync pays exactly the pre-scope + // write cost. Conservatively true is always safe; false with + // entries present is not. + // + // Transitions: probed at Open (ProbeSourceScopeMayExist), flipped + // true by stageSourceScopeChange the moment a stamped record is + // staged (self-healing — no caller coordination can be forgotten), + // armed by FoldBatch.Set when a raw compactor write actually stages + // a borrowed scope-index key the typed ops never see, re-probed by + // bulk import's Finish after its SST ingest (grantIndexKeys emits + // scope entries for stamped records; see the ingest-family + // obligation in families.go), and cleared only by the engine's + // ResetForNewSync wipe, which excises the index families wholesale. + // Deliberately NOT cleared by replay-state invalidation: a failed + // invalidation commit after a clear would leave false-with-entries, + // and stale-true only costs perf. + // + // Stamped primary rows WITHOUT index entries (the post-invalidation + // state on rebuild-compacted stores) keep the flag false at open: + // that is sound because everything the flag gates exists to + // maintain index entries, and there are none to maintain. + sourceScopeMayExist atomic.Bool + // testArmDeferredMarkerHook / testClearDeferredMarkerHook run // before the marker's durable commit / delete — the in-process // analogs of those writes failing. Installed only via // SetDeferredMarkerTestHooks (testing-gated). testArmDeferredMarkerHook func() error testClearDeferredMarkerHook func() error + testRecordCommitHook func() error + + // poisonObserver, when set, is invoked once per distinct poison + // event AFTER the batch that staged it commits (CO-015 requires + // poison events logged with scope, kind, and cause; the marker + // itself carries no value). Installed once by the engine at Open, + // before any concurrent use — not synchronized. + poisonObserver func(PoisonEvent) + + // acct is the ride-along resource ledger for family batches: + // every New*Batch increments its family counter and the batch's + // first Close decrements it, so Close can report an unreleased + // batch as a leak the same way pebble itself reports leaked + // iterators and Get closers (via readState/version refs). Pebble + // does not track batches, so this is the missing third of the + // resource-leak oracle. + acct batchAccounting +} + +// batchAccounting counts outstanding (minted, not yet closed) family +// batches. Always-on: the atomics are contention-free, and prod leak +// visibility matches pebble's own iterator accounting. +type batchAccounting struct { + record atomic.Int64 + session atomic.Int64 + digest atomic.Int64 + fold atomic.Int64 +} + +func (a *batchAccounting) leakError() error { + r, s, dg, f := a.record.Load(), a.session.Load(), a.digest.Load(), a.fold.Load() + if r == 0 && s == 0 && dg == 0 && f == 0 { + return nil + } + return fmt.Errorf("rawdb: unreleased family batches at Close: record=%d session=%d digest=%d fold=%d", r, s, dg, f) } // Open opens the pebble database at dir. opts is consumed by @@ -92,8 +157,13 @@ func Open(dir string, opts *pebble.Options, fs vfs.FS) (*DB, error) { } // Close closes the underlying pebble.DB. The engine's teardown -// ordering (write barrier, worker drain) is the caller's job. -func (d *DB) Close() error { return d.db.Close() } +// ordering (write barrier, worker drain) is the caller's job. Like +// pebble's own Close (which reports leaked iterators and Get closers +// through version refcounts), it reports any family batch minted but +// never released — the DB still closes; the error names the leak. +func (d *DB) Close() error { + return errors.Join(d.acct.leakError(), d.db.Close()) +} // FS returns the filesystem the DB's IO rides on. SSTs staged for // Ingest/IngestAndExcise must be created through it. @@ -232,6 +302,45 @@ func (d *DB) ProbeGrantDigestsPresent() error { return iter.Error() } +// SourceScopeMayExist reports whether any by_source_scope index entry +// may exist (the record ops' scope-obligation gate; see the field doc). +func (d *DB) SourceScopeMayExist() bool { return d.sourceScopeMayExist.Load() } + +// SetSourceScopeMayExist flips the scope-presence gate. The engine owns +// the false transition (ResetForNewSync, after the index families are +// excised); true transitions happen inside this package on staging. +func (d *DB) SetSourceScopeMayExist(present bool) { d.sourceScopeMayExist.Store(present) } + +// ProbeSourceScopeMayExist initializes the scope-presence gate with one +// bounded seek per by_source_scope family (the Open-time probe). +func (d *DB) ProbeSourceScopeMayExist() error { + for _, indexID := range []byte{ + IdxResourceBySourceScope, + IdxEntitlementBySourceScope, + IdxGrantBySourceScope, + } { + lo := []byte{VersionV3, TypeIndex, indexID} + iter, err := d.db.NewIter(&pebble.IterOptions{LowerBound: lo, UpperBound: UpperBound(lo)}) + if err != nil { + return err + } + found := iter.First() + if err := iter.Error(); err != nil { + iter.Close() + return err + } + if err := iter.Close(); err != nil { + return err + } + if found { + d.sourceScopeMayExist.Store(true) + return nil + } + } + d.sourceScopeMayExist.Store(false) + return nil +} + // SetDeferredMarkerTestHooks installs failure-injection hooks for the // marker's durable arm/clear. Test-only, same runtime gate as // UnsafeForTesting; pass nil to uninstall. @@ -243,6 +352,34 @@ func (d *DB) SetDeferredMarkerTestHooks(armHook, clearHook func() error) { d.testClearDeferredMarkerHook = clearHook } +// SetRecordCommitTestHook installs a failure immediately before every typed +// RecordBatch commit. It lets obligation tests cover ordinary resource, +// entitlement, and grant mutation paths through their shared choke point +// instead of adding incomplete engine-path-specific seams. Test-only; pass nil +// to uninstall. +func (d *DB) SetRecordCommitTestHook(hook func() error) { + if !testing.Testing() { + panic("rawdb.SetRecordCommitTestHook: called outside a test binary") + } + d.testRecordCommitHook = hook +} + +// PoisonEvent describes one staged source-cache poison marker: the +// (row_kind, scope) that lost a row and why. Delivered to the poison +// observer after the staging batch commits. +type PoisonEvent struct { + RowKind string + ScopeKey string + Cause string +} + +// SetPoisonObserver installs the post-commit poison event callback. +// Must be called before the DB sees concurrent use (the engine installs +// it at Open); pass nil to uninstall. +func (d *DB) SetPoisonObserver(fn func(PoisonEvent)) { + d.poisonObserver = fn +} + // === lifecycle operations (write-class, engine-lifecycle-named) === // FlushMemtables forces the memtable out to L0 (pebble.DB.Flush, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/records.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/records.go index 7da733df..66044b6f 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/records.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb/records.go @@ -16,7 +16,7 @@ package rawdb // - INLINE (StageGrantPutInline / StageGrantDelete): by_principal // and by_needs_expansion maintained inline; overwrite/delete // cleans BOTH; digest invalidation when digests are present. The -// PutGrantRecords and IfNewer paths, and post-seal deletes. +// PutGrantRecords paths and post-seal deletes. // - DEFERRED (StageGrantPutDeferred): the durable deferred-index // marker is armed FIRST (ArmDeferredGrantIndex — CAS-cheap per // record, crash-contract-ordered before the batch can commit), @@ -39,6 +39,8 @@ package rawdb import ( "fmt" + + "google.golang.org/protobuf/encoding/protowire" ) // Family prefixes for the keyspace assertions. @@ -56,6 +58,32 @@ func assertFamily(op string, key, prefix []byte) error { return nil } +// StageSourceCacheReplayInvalidation removes validators that cannot describe a +// compaction output. Rebuild compactions also drop all source-scope indexes; +// fold compactions retain them to avoid rewriting the inherited base. +// The wipe covers the whole source-cache family — poison markers go with +// the manifest entries they qualify (no entry, nothing to refuse). +func (rb *RecordBatch) StageSourceCacheReplayInvalidation(dropScopeIndexes bool) error { + lo, hi := SourceCacheFamilyBounds() + if err := rb.core.DeleteRange(lo, hi); err != nil { + return err + } + if !dropScopeIndexes { + return nil + } + for _, indexID := range []byte{ + IdxResourceBySourceScope, + IdxEntitlementBySourceScope, + IdxGrantBySourceScope, + } { + lo := []byte{VersionV3, TypeIndex, indexID} + if err := rb.core.DeleteRange(lo, UpperBound(lo)); err != nil { + return err + } + } + return nil +} + // === grant staging === // StageGrantPutInline stages one grant row in the INLINE index regime @@ -68,7 +96,7 @@ func assertFamily(op string, key, prefix []byte) error { // (the write path has it; deriving it here would force a value scan). // Contrast StageResourcePut, which takes the prior VALUE bytes — // resource index keys derive from the value (parent ref), not the key. -func (rb *RecordBatch) StageGrantPutInline(key, val []byte, hadOldVal, needsExpansion bool) error { +func (rb *RecordBatch) StageGrantPutInline(key, val, oldVal []byte, needsExpansion bool) error { if err := assertFamily("StageGrantPutInline", key, grantPrimaryPrefix); err != nil { return err } @@ -76,7 +104,7 @@ func (rb *RecordBatch) StageGrantPutInline(key, val []byte, hadOldVal, needsExpa if !ok { return fmt.Errorf("rawdb.StageGrantPutInline: grant key %x did not decode as a 6-segment identity", key) } - if hadOldVal { + if oldVal != nil { if err := rb.deleteByPrincipalKey(key); err != nil { return err } @@ -95,6 +123,9 @@ func (rb *RecordBatch) StageGrantPutInline(key, val []byte, hadOldVal, needsExpa return err } } + if err := rb.stageSourceScopeChange(key, val, oldVal, 10); err != nil { + return err + } return rb.stageGrantDigestInvalidation(key, sep4) } @@ -110,7 +141,7 @@ func (rb *RecordBatch) StageGrantPutInline(key, val []byte, hadOldVal, needsExpa // a stale-but-present digest, a present-means-exact hole. Key-derived // cleanup cannot be skipped; tombstones on absent index keys are // harmless. -func (rb *RecordBatch) StageGrantDelete(key []byte) error { +func (rb *RecordBatch) StageGrantDelete(key, oldVal []byte) error { if err := assertFamily("StageGrantDelete", key, grantPrimaryPrefix); err != nil { return err } @@ -127,6 +158,9 @@ func (rb *RecordBatch) StageGrantDelete(key []byte) error { if err := rb.core.b.Delete(key, nil); err != nil { return err } + if err := rb.stageSourceScopeCleanup(key, oldVal, 10); err != nil { + return err + } return rb.stageGrantDigestInvalidation(key, sep4) } @@ -138,7 +172,7 @@ func (rb *RecordBatch) StageGrantDelete(key []byte) error { // wholesale at seal, which also clears stale entries), and the digest // invalidation is staged. hadOldVal selects overwrite cleanup of the // needs_expansion entry. -func (rb *RecordBatch) StageGrantPutDeferred(key, val []byte, hadOldVal, needsExpansion bool) error { +func (rb *RecordBatch) StageGrantPutDeferred(key, val, oldVal []byte, needsExpansion bool) error { if err := assertFamily("StageGrantPutDeferred", key, grantPrimaryPrefix); err != nil { return err } @@ -149,7 +183,7 @@ func (rb *RecordBatch) StageGrantPutDeferred(key, val []byte, hadOldVal, needsEx if err := rb.db.ArmDeferredGrantIndex(); err != nil { return err } - if hadOldVal { + if oldVal != nil { if err := rb.deleteNeedsExpansionKey(key); err != nil { return err } @@ -162,6 +196,9 @@ func (rb *RecordBatch) StageGrantPutDeferred(key, val []byte, hadOldVal, needsEx return err } } + if err := rb.stageSourceScopeChange(key, val, oldVal, 10); err != nil { + return err + } return rb.stageGrantDigestInvalidation(key, sep4) } @@ -188,6 +225,21 @@ func (rb *RecordBatch) StageGrantOrphanIndexHeal(primaryKey []byte) error { return rb.deleteByPrincipalKey(primaryKey) } +// StageSourceScopeOrphanIndexDelete removes one by_source_scope entry whose +// primary row is absent. The caller must establish absence while holding the +// engine write barrier. +func (rb *RecordBatch) StageSourceScopeOrphanIndexDelete(indexKey []byte) error { + if len(indexKey) < 3 || indexKey[0] != VersionV3 || indexKey[1] != TypeIndex { + return fmt.Errorf("rawdb.StageSourceScopeOrphanIndexDelete: key %x is not an index key", indexKey) + } + switch indexKey[2] { + case IdxGrantBySourceScope, IdxEntitlementBySourceScope, IdxResourceBySourceScope: + return rb.core.b.Delete(indexKey, nil) + default: + return fmt.Errorf("rawdb.StageSourceScopeOrphanIndexDelete: key %x is outside source-scope families", indexKey) + } +} + func (rb *RecordBatch) setByPrincipalKey(key []byte) error { idx, ok := AppendGrantByPrincipalKeyFromPrimary(rb.scratch[:0], key) rb.scratch = idx @@ -278,9 +330,12 @@ func (rb *RecordBatch) StageResourcePut(key, val, oldVal []byte, childRT, childI return err } if parentID == "" { - return nil + return rb.stageSourceScopeChange(key, val, oldVal, 12) + } + if err := rb.core.b.Set(EncodeResourceByParentIndexKey(parentRT, parentID, childRT, childID), nil, nil); err != nil { + return err } - return rb.core.b.Set(EncodeResourceByParentIndexKey(parentRT, parentID, childRT, childID), nil, nil) + return rb.stageSourceScopeChange(key, val, oldVal, 12) } // StageResourceDelete stages one resource row's removal plus its @@ -292,6 +347,9 @@ func (rb *RecordBatch) StageResourceDelete(key, oldVal []byte, childRT, childID if err := rb.stageResourceParentDelete(oldVal, childRT, childID); err != nil { return err } + if err := rb.stageSourceScopeCleanup(key, oldVal, 12); err != nil { + return err + } return rb.core.b.Delete(key, nil) } @@ -315,21 +373,160 @@ func (rb *RecordBatch) stageResourceParentDelete(oldVal []byte, childRT, childID // obligation and stays with the engine's write paths. // StageEntitlementPut stages one entitlement row. -func (rb *RecordBatch) StageEntitlementPut(key, val []byte) error { +func (rb *RecordBatch) StageEntitlementPut(key, val, oldVal []byte) error { if err := assertFamily("StageEntitlementPut", key, entitlementPrimaryPrefix); err != nil { return err } - return rb.core.b.Set(key, val, nil) + if err := rb.core.b.Set(key, val, nil); err != nil { + return err + } + return rb.stageSourceScopeChange(key, val, oldVal, 11) } // StageEntitlementDelete stages one entitlement row's removal. -func (rb *RecordBatch) StageEntitlementDelete(key []byte) error { +func (rb *RecordBatch) StageEntitlementDelete(key, oldVal []byte) error { if err := assertFamily("StageEntitlementDelete", key, entitlementPrimaryPrefix); err != nil { return err } + if err := rb.stageSourceScopeCleanup(key, oldVal, 11); err != nil { + return err + } return rb.core.b.Delete(key, nil) } +// stageSourceScopeCleanup stages the by_source_scope entry removal a +// record DELETE owes, gated on sourceScopeMayExist exactly like +// stageSourceScopeChange: an unarmed gate certifies the index families +// are empty, so there is no entry to remove and the prior-value scan is +// skipped. A malformed prior value fails the delete rather than dropping +// the primary and stranding its index — the same policy the put path +// applies to the same unreadable bytes, so a corrupt row is uniformly +// immovable instead of deletable-but-not-overwritable. +// +// Deleting a stamped row on behalf of any actor OTHER than its own +// scope (actingSourceScope, default unscoped — interactive deletes, +// external-principal reconciliation, SDK maintenance passes) also +// stages the scope's poison marker (CO-015): the scope's manifest entry +// vouches for a row set this delete just shrank, so a later 304-replay +// of the scope would silently drop the row. A scope's own tombstone +// paths set actingSourceScope and stay poison-free — shrinking yourself +// is the legitimate delta flow, and the manifest validator rotates with +// it. +func (rb *RecordBatch) stageSourceScopeCleanup(key, oldVal []byte, field protowire.Number) error { + if !rb.db.sourceScopeMayExist.Load() { + return nil + } + oldScope, err := ScanSourceScopeKeyRaw(oldVal, field) + if err != nil { + return err + } + if oldScope == "" { + return nil + } + if oldScope != rb.actingSourceScope { + if err := rb.stageSourceScopePoison(key, oldScope, "row deleted by an actor outside its scope"); err != nil { + return err + } + } + return rb.deleteSourceScopeKey(key, oldScope) +} + +// stageSourceScopeChange stages the by_source_scope index obligations a +// record put owes. The new value is ALWAYS scanned (an O(#fields) header +// walk), so a stamped record arms the sourceScopeMayExist gate right here +// — no caller can forget to. The PRIOR value is scanned only when the +// gate was already armed: an unarmed gate certifies the index families +// are empty, so a stale stamp in oldVal has no entry to clean up (and +// callers on that fast path may skip fetching oldVal entirely — see +// PutEntitlementRecords). +func (rb *RecordBatch) stageSourceScopeChange(key, val, oldVal []byte, field protowire.Number) error { + mayExist := rb.db.sourceScopeMayExist.Load() + newScope, err := ScanSourceScopeKeyRaw(val, field) + if err != nil { + return err + } + if newScope == "" && !mayExist { + return nil + } + if mayExist { + oldScope, err := ScanSourceScopeKeyRaw(oldVal, field) + if err != nil { + return err + } + if oldScope != "" && oldScope != newScope { + // Cross-scope restamp (including a stamp-clearing unscoped + // overwrite): the old scope silently loses this row, so it is + // poisoned (CO-015). Detection is order-independent — whichever + // scope writes second observes the first's stamp in oldVal. + if err := rb.stageSourceScopePoison(key, oldScope, "cross-scope restamp"); err != nil { + return err + } + if err := rb.deleteSourceScopeKey(key, oldScope); err != nil { + return err + } + } + } + if newScope == "" { + return nil + } + rb.db.sourceScopeMayExist.Store(true) + indexKey, ok := AppendBySourceScopeKeyFromPrimary(rb.scratch[:0], key, newScope) + rb.scratch = indexKey + if !ok { + return fmt.Errorf("rawdb: cannot derive source-scope index from key %x", key) + } + return rb.core.b.Set(indexKey, nil, nil) +} + +func (rb *RecordBatch) deleteSourceScopeKey(key []byte, scopeKey string) error { + indexKey, ok := AppendBySourceScopeKeyFromPrimary(rb.scratch[:0], key, scopeKey) + rb.scratch = indexKey + if !ok { + return fmt.Errorf("rawdb: cannot derive source-scope index from key %x", key) + } + return rb.core.b.Delete(indexKey, nil) +} + +// stageSourceScopePoison stages the durable poison marker for the scope +// that just lost the row at key, rides the same batch as the losing +// mutation (atomic: no crash image holds the loss without the verdict), +// and records the event for post-commit observer delivery. Staging is +// deduplicated per BATCH only — a page of restamps against one scope +// stages one marker and delivers one event, but batches re-mint per +// chunk, so the observer sees one event per chunk that poisons the +// scope. Log-level dedup (once per (kind, scope) per open) lives in the +// engine's observer, which owns cross-batch state. +func (rb *RecordBatch) stageSourceScopePoison(key []byte, lostScope, cause string) error { + rowKind, ok := RowKindForRecordType(key[1]) + if !ok { + return fmt.Errorf("rawdb: poison: key %x is not a record primary", key) + } + ev := PoisonEvent{RowKind: rowKind, ScopeKey: lostScope, Cause: cause} + if rb.poisonStaged == nil { + rb.poisonStaged = make(map[PoisonEvent]struct{}) + } + if _, dup := rb.poisonStaged[ev]; dup { + return nil + } + if err := rb.core.b.Set(SourceCachePoisonKey(rowKind, lostScope), nil, nil); err != nil { + return err + } + rb.poisonStaged[ev] = struct{}{} + rb.poisonEvents = append(rb.poisonEvents, ev) + return nil +} + +// SetActingSourceScope declares the scope on whose behalf this batch's +// DELETES act. A delete of a row stamped with the acting scope is that +// scope shrinking itself (the legitimate delta-tombstone and +// replay-replacement flows) and stages no poison; every other stamped +// delete poisons the row's scope. The default — unscoped — is the +// conservative setting: an actor that has not identified itself poisons +// whatever stamped rows it removes. +func (rb *RecordBatch) SetActingSourceScope(scope string) { + rb.actingSourceScope = scope +} + // StageResourceTypePut stages one resource-type row. func (rb *RecordBatch) StageResourceTypePut(key, val []byte) error { if err := assertFamily("StageResourceTypePut", key, resourceTypePrimaryPrefix); err != nil { @@ -358,9 +555,37 @@ func (rb *RecordBatch) StageResourceTypeDelete(key []byte) error { // fold compactor. type FoldBatch struct { batch + db *DB } // NewFoldBatch mints a generic staged batch for the compactor's // keep-newer fold and overlay writers. Engine production code must // not use it; see the choke-point meta-tests. -func (d *DB) NewFoldBatch() *FoldBatch { return &FoldBatch{batch{b: d.newBatch()}} } +// +// A fold destination inherited from a scoped base is already armed by +// ProbeSourceScopeMayExist at Open. Rebuild paths start empty and must not pay +// source-scope maintenance merely because they share this raw batch surface. +// FoldBatch.Set therefore arms the gate only when a caller actually stages a +// by_source_scope key. +func (d *DB) NewFoldBatch() *FoldBatch { + d.acct.fold.Add(1) + return &FoldBatch{ + batch: batch{b: d.newBatch(), open: &d.acct.fold}, + db: d, + } +} + +// Set stages a raw compactor key and self-arms the source-scope obligation +// gate exactly when the raw surface introduces an entry that typed record ops +// cannot observe. +func (b *FoldBatch) Set(key, val []byte) error { + if len(key) >= 3 && + key[0] == VersionV3 && + key[1] == TypeIndex { + switch key[2] { + case IdxResourceBySourceScope, IdxEntitlementBySourceScope, IdxGrantBySourceScope: + b.db.sourceScopeMayExist.Store(true) + } + } + return b.batch.Set(key, val) +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go index 16ffe94d..23e1b282 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/keys.go @@ -68,6 +68,7 @@ const ( typeCounter = rawdb.TypeCounter typeSession = rawdb.TypeSession typeDigest = rawdb.TypeDigest + typeSourceCache = rawdb.TypeSourceCache typeEngineMeta = rawdb.TypeEngineMeta ) @@ -80,6 +81,9 @@ const ( idxGrantByPrincipalResourceType = rawdb.IdxGrantByPrincipalResourceType idxGrantByEntitlementResource = rawdb.IdxGrantByEntitlementResource idxGrantByEntitlementPrincipalHash = rawdb.IdxGrantByEntitlementPrincipalHash + idxGrantBySourceScope = rawdb.IdxGrantBySourceScope + idxEntitlementBySourceScope = rawdb.IdxEntitlementBySourceScope + idxResourceBySourceScope = rawdb.IdxResourceBySourceScope ) // --- Grant --- @@ -272,6 +276,38 @@ func encodeGrantByPrincipalResourceTypeIdentityPrefix(principalRT string) []byte return codec.AppendTupleSeparator(buf) } +// --- Source-cache scope indexes and manifest --- + +func encodeGrantBySourceScopeIndexKey(scopeKey string, id grantIdentity) []byte { + // Callers only enter with a non-empty scope, and encodeGrantIdentityKey + // always emits the v3 grant-primary shape accepted by the raw helper. + // Therefore ok is an invariant here rather than recoverable input. + key, ok := rawdb.AppendBySourceScopeKeyFromPrimary(nil, encodeGrantIdentityKey(id), scopeKey) + if !ok { + panic("pebble: invalid grant source-scope index key invariant") + } + return key +} + +func encodeGrantBySourceScopePrefix(scopeKey string) []byte { + prefix, _ := rawdb.SourceScopeIndexPrefix(typeGrant, scopeKey) + return prefix +} + +func encodeEntitlementBySourceScopePrefix(scopeKey string) []byte { + prefix, _ := rawdb.SourceScopeIndexPrefix(typeEntitlement, scopeKey) + return prefix +} + +func encodeResourceBySourceScopePrefix(scopeKey string) []byte { + prefix, _ := rawdb.SourceScopeIndexPrefix(typeResource, scopeKey) + return prefix +} + +func encodeSourceCacheEntryKey(rowKind, scopeKey string) []byte { + return rawdb.SourceCacheEntryKey(rowKind, scopeKey) +} + // --- Grant by (entitlement, principal-hash) + digest nodes --- // // PARTITION CONVENTION. Both keyspaces below are addressed by a digest @@ -347,6 +383,43 @@ func DigestUpperBound() []byte { return upperBoundOf(DigestLowerBound()) } +func GrantBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxGrantBySourceScope} +} +func GrantBySourceScopeUpperBound() []byte { return upperBoundOf(GrantBySourceScopeLowerBound()) } + +func EntitlementBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxEntitlementBySourceScope} +} +func EntitlementBySourceScopeUpperBound() []byte { + return upperBoundOf(EntitlementBySourceScopeLowerBound()) +} + +func ResourceBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxResourceBySourceScope} +} +func ResourceBySourceScopeUpperBound() []byte { + return upperBoundOf(ResourceBySourceScopeLowerBound()) +} + +func SourceCacheEntryLowerBound() []byte { + lo, _ := rawdb.SourceCacheEntryBounds() + return lo +} +func SourceCacheEntryUpperBound() []byte { + _, hi := rawdb.SourceCacheEntryBounds() + return hi +} + +func SourceCachePoisonLowerBound() []byte { + lo, _ := rawdb.SourceCachePoisonBounds() + return lo +} +func SourceCachePoisonUpperBound() []byte { + _, hi := rawdb.SourceCachePoisonBounds() + return hi +} + // --- ResourceType --- // encodeResourceTypeKey returns the primary key for a resource_type: diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/lookup.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/lookup.go index 2b70e59f..43f5ae07 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/lookup.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/lookup.go @@ -265,6 +265,17 @@ func (e *Engine) grantPrimaryPrefixNonEmpty(prefix []byte) (bool, error) { // the row instead of the concat). Exactly one hit wins; zero is // pebble.ErrNotFound; several is ErrAmbiguousExternalID. func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID string) (grantIdentity, error) { + return e.resolveGrantIdentity(ctx, grantID, true) +} + +// resolveGrantIdentityByCandidates is the bounded variant used by +// source-cache tombstones. It never falls back to an O(all grants) scan for +// connector-custom stored ids. +func (e *Engine) resolveGrantIdentityByCandidates(ctx context.Context, grantID string) (grantIdentity, error) { + return e.resolveGrantIdentity(ctx, grantID, false) +} + +func (e *Engine) resolveGrantIdentity(ctx context.Context, grantID string, allowStoredIDScan bool) (grantIdentity, error) { var colons []int for i := 0; i < len(grantID); i++ { if grantID[i] == ':' { @@ -275,7 +286,10 @@ func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID s // No concat shape to split: connector-custom ids (SQLite keyed rows // by these, and provisioner revokes address grants with them) are // findable only by their STORED external id. - return e.scanGrantIdentityByStoredExternalID(ctx, grantID) + if allowStoredIDScan { + return e.scanGrantIdentityByStoredExternalID(ctx, grantID) + } + return grantIdentity{}, pebble.ErrNotFound } if len(colons) > maxBareIDColons { return grantIdentity{}, fmt.Errorf("%w: grant id has %d colons; too complex to resolve safely by string", ErrAmbiguousExternalID, len(colons)) @@ -380,7 +394,10 @@ func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID s case 0: // Every concat split missed: the id may still be a connector-custom // STORED external id that merely contains colons. - return e.scanGrantIdentityByStoredExternalID(ctx, grantID) + if allowStoredIDScan { + return e.scanGrantIdentityByStoredExternalID(ctx, grantID) + } + return grantIdentity{}, pebble.ErrNotFound case 1: return hits[0], nil default: diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/manifest.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/manifest.go index 8a11bd48..bfa24a9e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/manifest.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/manifest.go @@ -87,6 +87,7 @@ func syncRunSummaries(ctx context.Context, e *Engine) ([]*c1zv3.SyncRunSummary, EndedAt: r.GetEndedAt(), ParentSyncId: r.GetParentSyncId(), Stats: stats, + Compacted: r.GetCompacted(), }.Build()) return true }) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go index a3db1e82..dd92a6b7 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/raw_records.go @@ -2,114 +2,12 @@ package pebble import ( "fmt" - "math" - "time" "google.golang.org/protobuf/encoding/protowire" - "google.golang.org/protobuf/types/known/timestamppb" "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb" ) -const ( - resourceTypeDiscoveredAtField protowire.Number = 6 - resourceDiscoveredAtField protowire.Number = 8 - entitlementDiscoveredAtField protowire.Number = 8 - grantDiscoveredAtField protowire.Number = 5 -) - -func discoveredAtIsNewerThanRaw(incoming *timestamppb.Timestamp, existingValue []byte, field protowire.Number) (bool, error) { - if incoming == nil { - return false, nil - } - existing, ok, err := rawDiscoveredAtNanos(existingValue, field) - if err != nil { - return false, err - } - if !ok { - return true, nil - } - return incoming.AsTime().UnixNano() > existing, nil -} - -func rawDiscoveredAtNanos(value []byte, field protowire.Number) (int64, bool, error) { - for len(value) > 0 { - num, typ, n := protowire.ConsumeTag(value) - if n < 0 { - return 0, false, protowire.ParseError(n) - } - value = value[n:] - if num != field { - n = protowire.ConsumeFieldValue(num, typ, value) - if n < 0 { - return 0, false, protowire.ParseError(n) - } - value = value[n:] - continue - } - if typ != protowire.BytesType { - return 0, false, fmt.Errorf("raw record: discovered_at has wire type %v", typ) - } - ts, n := protowire.ConsumeBytes(value) - if n < 0 { - return 0, false, protowire.ParseError(n) - } - nanos, err := rawTimestampNanos(ts) - return nanos, true, err - } - return 0, false, nil -} - -func rawTimestampNanos(value []byte) (int64, error) { - var seconds int64 - var nanos int32 - for len(value) > 0 { - num, typ, n := protowire.ConsumeTag(value) - if n < 0 { - return 0, protowire.ParseError(n) - } - value = value[n:] - switch num { - case 1: - if typ != protowire.VarintType { - return 0, fmt.Errorf("raw record: timestamp seconds has wire type %v", typ) - } - v, n := protowire.ConsumeVarint(value) - if n < 0 { - return 0, protowire.ParseError(n) - } - if v > math.MaxInt64 { - return 0, fmt.Errorf("raw record: timestamp seconds exceeds int64: %d", v) - } - seconds = int64(v) - value = value[n:] - case 2: - if typ != protowire.VarintType { - return 0, fmt.Errorf("raw record: timestamp nanos has wire type %v", typ) - } - v, n := protowire.ConsumeVarint(value) - if n < 0 { - return 0, protowire.ParseError(n) - } - if v > math.MaxInt32 { - return 0, fmt.Errorf("raw record: timestamp nanos exceeds int32: %d", v) - } - nanos = int32(v) - value = value[n:] - default: - n = protowire.ConsumeFieldValue(num, typ, value) - if n < 0 { - return 0, protowire.ParseError(n) - } - value = value[n:] - } - } - if seconds > math.MaxInt64/int64(time.Second) { - return 0, fmt.Errorf("raw record: timestamp seconds overflow: %d", seconds) - } - return seconds*int64(time.Second) + int64(nanos), nil -} - // NOTE (2b): deleteResourceIndexesRaw / deleteGrantIndexesRaw are GONE. // Prior-row index cleanup is an obligation of rawdb's typed record ops // (StageGrantPutInline/StageGrantDelete derive cleanup keys from the diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/source_cache.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/source_cache.go new file mode 100644 index 00000000..f945e4c3 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/source_cache.go @@ -0,0 +1,1783 @@ +package pebble + +import ( + "context" + "errors" + "fmt" + "io" + + "github.com/cockroachdb/pebble/v2" + "google.golang.org/protobuf/encoding/protowire" + "google.golang.org/protobuf/types/known/timestamppb" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// Source-cache replay, engine side. +// +// The typeSourceCache keyspace holds one SourceCacheEntryRecord per +// (row_kind, scope_hash): the opaque upstream validator (etag / delta +// token) the sync recorded for that scope. Rows produced under a scope +// are stamped with source_scope_key and indexed under the +// by_source_scope families, whose tails are identity tuples — so a +// replay derives every primary key from the index key alone and copies +// raw values across files without a proto unmarshal. +// +// The previous sync lives in a separate read-only engine (a Pebble c1z +// holds exactly one sync); replay copies from prev into the receiver. + +// replayBatchRows bounds how many rows accumulate in one pebble.Batch +// before an intermediate commit. Replay of a delta-query collection can +// be the whole previous row set, so the batch must not grow unbounded. +const replayBatchRows = 10_000 + +// manifestRewritePageRows bounds one page of a seal/clear manifest +// rewrite. Manifest size is (scopes × row kinds) with connector-chosen +// scope granularity, so these passes page in the same dimension and at +// the same bound as every other scope-scale pass. +const manifestRewritePageRows = replayBatchRows + +func (e *Engine) sourceCacheReplayBatchLimit() int { + if e.test.sourceCacheReplayBatchRows > 0 { + return e.test.sourceCacheReplayBatchRows + } + return replayBatchRows +} + +func (e *Engine) sourceCacheReplayIteratorError(kind string, iter *pebble.Iterator) error { + if e.test.sourceCacheReplayIteratorErrorHook != nil { + if err := e.test.sourceCacheReplayIteratorErrorHook(kind); err != nil { + return err + } + } + return iter.Error() +} + +// SourceCacheReplayResult reports what one scope's replay copied. +// +// The result is meaningful WITH a non-nil error too: bounded +// intermediate batches may have landed before the failure (that is the +// committed-prefix retry seam), so on error Rows reports only rows +// whose commit landed, matching the scoped-delete siblings' committed +// progress. NeedsExpansion accumulates at stage time and on error may +// overreport a row that never committed — the safe direction, since +// arming expansion is idempotent and add-only, while underreporting +// could leave a committed expandable grant unexpanded. +type SourceCacheReplayResult struct { + Rows int64 + // NeedsExpansion is true when at least one copied grant row carried + // needs_expansion. Future syncer replay orchestration must consume this + // signal to arm grant expansion: replayed pages never pass + // GrantExpandable-annotated rows through the connector-response path. + NeedsExpansion bool +} + +// sourceCacheRowKindSpecs maps each manifest row kind to its primary +// record family and the SourceScopeKey field number in that family's +// value proto. The field numbers are pinned by +// TestVerificationDescriptorClosedReplayAndDirectMaterialization. +var sourceCacheRowKindSpecs = map[string]struct { + recordType byte + scopeField protowire.Number +}{ + string(sourcecache.RowKindResources): {typeResource, 12}, + string(sourcecache.RowKindEntitlements): {typeEntitlement, 11}, + string(sourcecache.RowKindGrants): {typeGrant, 10}, +} + +// sealSourceCacheRowCounts stamps every manifest entry with the number +// of primary rows carrying its scope, recomputed from the primary +// keyspace at seal time (CO-004). Replay preflight compares the scope's +// index cardinality against this count instead of scanning every +// primary per scope, turning the preflight from O(scopes × rows) into +// O(scope size). +// +// Counting from the PRIMARIES (not the by_source_scope index) is the +// point: the count must be an independent witness of the same +// biconditional the old preflight proved, so index corruption cannot +// vouch for itself. Zero is written explicitly (field presence) — a +// proven-empty scope is a valid replay source, while an entry missing +// the count entirely means the artifact predates or skipped this seal +// step and replay must hard-fail. +// +// Runs SEALED, before the ended_at stamp: a crash between counting and +// the stamp leaves the sync unfinished, so no artifact can carry the +// finished verdict without also carrying the counts (they ride the +// same WAL the stamp fsyncs). Reseal after a SetCurrentSync rebind +// recounts and overwrites — rebound mutations never publish stale +// counts, and an unpublished working directory is never a replay +// source. Cost is zero for syncs with an empty manifest. +func (e *Engine) sealSourceCacheRowCounts(ctx context.Context) error { + return e.withWriteAllowSealed(func() error { + lo, hi := rawdb.SourceCacheEntryBounds() + // Pass 1 — validate every entry's kind and learn which primary + // families need counting. Entries are NOT retained: manifest + // size is (scopes × row kinds) and scope granularity is + // connector-chosen, so the only cross-pass state this function + // holds is the counts map — per-scope totals, which are the + // output and thus an irreducible O(scopes-with-stamped-rows). + iter, err := e.db.NewIter(&pebble.IterOptions{LowerBound: lo, UpperBound: hi}) + if err != nil { + return fmt.Errorf("seal source cache counts: manifest iter: %w", err) + } + kinds := make(map[string]struct{}) + hasEntries := false + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + _ = iter.Close() + return err + } + rec := &v3.SourceCacheEntryRecord{} + if err := unmarshalRecord(iter.Value(), rec); err != nil { + _ = iter.Close() + return fmt.Errorf("seal source cache counts: manifest %x: %w", iter.Key(), err) + } + if _, ok := sourceCacheRowKindSpecs[rec.GetRowKind()]; !ok { + _ = iter.Close() + return fmt.Errorf("seal source cache counts: manifest %x has unknown row kind %q", iter.Key(), rec.GetRowKind()) + } + hasEntries = true + kinds[rec.GetRowKind()] = struct{}{} + } + if err := iter.Error(); err != nil { + _ = iter.Close() + return fmt.Errorf("seal source cache counts: manifest iter: %w", err) + } + if err := iter.Close(); err != nil { + return err + } + if !hasEntries { + return nil + } + + // One pass per row kind that has manifest entries, regardless of + // how many scopes partition it. + counts := make(map[string]map[string]uint64, len(kinds)) + for kind := range kinds { + spec := sourceCacheRowKindSpecs[kind] + kindCounts := make(map[string]uint64) + primaryPrefix := []byte{versionV3, spec.recordType} + primaries, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: primaryPrefix, + UpperBound: upperBoundOf(primaryPrefix), + }) + if err != nil { + return fmt.Errorf("seal source cache counts: %s primaries: %w", kind, err) + } + var scanned int + for primaries.First(); primaries.Valid(); primaries.Next() { + scanned++ + if scanned&0x3FF == 0 { + if err := ctx.Err(); err != nil { + _ = primaries.Close() + return err + } + } + stamp, err := rawdb.ScanSourceScopeKeyRaw(primaries.Value(), spec.scopeField) + if err != nil { + _ = primaries.Close() + return fmt.Errorf("seal source cache counts: %s primary %x: %w", kind, primaries.Key(), err) + } + if stamp == "" { + continue + } + kindCounts[stamp]++ + } + if err := primaries.Error(); err != nil { + _ = primaries.Close() + return fmt.Errorf("seal source cache counts: %s primaries: %w", kind, err) + } + if err := primaries.Close(); err != nil { + return err + } + counts[kind] = kindCounts + } + + // Pass 3 — stream the counted entries back in bounded pages. The + // iterator reads the pre-write snapshot, so rewriting keys it has + // already visited cannot disturb the traversal, and only one page + // is ever resident. NoSync: these writes ride the WAL ahead of + // the ended_at stamp's fsync, so any crash image holding the + // finished verdict also holds every count (same argument as + // endSyncFinalize's page hardening). + rewrite, err := e.db.NewIter(&pebble.IterOptions{LowerBound: lo, UpperBound: hi}) + if err != nil { + return fmt.Errorf("seal source cache counts: manifest rewrite iter: %w", err) + } + var page []rawdb.SourceCacheKV + flush := func() error { + if err := e.db.SourceCacheSetMulti(page, pebble.NoSync); err != nil { + return fmt.Errorf("seal source cache counts: write: %w", err) + } + page = page[:0] + return nil + } + for rewrite.First(); rewrite.Valid(); rewrite.Next() { + if err := ctx.Err(); err != nil { + _ = rewrite.Close() + return err + } + rec := &v3.SourceCacheEntryRecord{} + if err := unmarshalRecord(rewrite.Value(), rec); err != nil { + _ = rewrite.Close() + return fmt.Errorf("seal source cache counts: manifest %x: %w", rewrite.Key(), err) + } + rec.SetRowCount(counts[rec.GetRowKind()][rec.GetScopeKey()]) + val, err := marshalRecord(rec) + if err != nil { + _ = rewrite.Close() + return fmt.Errorf("seal source cache counts: marshal %x: %w", rewrite.Key(), err) + } + if len(page) >= manifestRewritePageRows { + if err := flush(); err != nil { + _ = rewrite.Close() + return err + } + } + page = append(page, rawdb.SourceCacheKV{Key: append([]byte(nil), rewrite.Key()...), Val: val}) + } + if err := rewrite.Error(); err != nil { + _ = rewrite.Close() + return fmt.Errorf("seal source cache counts: manifest rewrite iter: %w", err) + } + if err := rewrite.Close(); err != nil { + return err + } + return flush() + }) +} + +// clearSourceCacheRowCounts strips sealed row counts from every manifest +// entry that carries one. Called on rebind (bindCurrentSync): a rebound +// sync admits new mutations, so counts sealed before those mutations no +// longer witness the primary keyspace. Published artifacts are unaffected +// (publication always passes through EndSync, which recounts); this keeps +// any UNPUBLISHED rebound store fail-closed — a replay-eligible entry +// without a count is a hard preflight error. No-op for unfinished syncs +// (counts exist only after a sealing EndSync) and empty manifests. +func (e *Engine) clearSourceCacheRowCounts() error { + return e.withWrite(func() error { + lo, hi := rawdb.SourceCacheEntryBounds() + iter, err := e.db.NewIter(&pebble.IterOptions{LowerBound: lo, UpperBound: hi}) + if err != nil { + return fmt.Errorf("clear source cache counts: manifest iter: %w", err) + } + // Paged rewrite over a snapshot iterator (writes to visited keys + // cannot disturb the traversal): intermediate pages commit NoSync + // and the FINAL page commits synced, which persists the whole WAL + // prefix — every earlier page is durable before bind returns, so + // no crash image holds a post-rebind mutation alongside any + // still-counted entry. A crash MID-strip is safe in both + // directions: bind never returned, so no mutation was admitted — + // stripped entries are fail-closed (a missing count is a hard + // preflight error) and still-counted entries remain accurate + // witnesses of the untouched primary keyspace; the reopen's bind + // strips the rest. Paging bounds memory and batch size by the + // page, not the manifest. + var page []rawdb.SourceCacheKV + flush := func(o *pebble.WriteOptions) error { + if err := e.db.SourceCacheSetMulti(page, o); err != nil { + return fmt.Errorf("clear source cache counts: write: %w", err) + } + page = page[:0] + return nil + } + cleared := false + for iter.First(); iter.Valid(); iter.Next() { + rec := &v3.SourceCacheEntryRecord{} + if err := unmarshalRecord(iter.Value(), rec); err != nil { + _ = iter.Close() + return fmt.Errorf("clear source cache counts: manifest %x: %w", iter.Key(), err) + } + if !rec.HasRowCount() { + continue + } + rec.ClearRowCount() + val, err := marshalRecord(rec) + if err != nil { + _ = iter.Close() + return fmt.Errorf("clear source cache counts: marshal %x: %w", iter.Key(), err) + } + if len(page) >= manifestRewritePageRows { + if err := flush(pebble.NoSync); err != nil { + _ = iter.Close() + return err + } + } + page = append(page, rawdb.SourceCacheKV{Key: append([]byte(nil), iter.Key()...), Val: val}) + cleared = true + } + if err := iter.Error(); err != nil { + _ = iter.Close() + return fmt.Errorf("clear source cache counts: manifest iter: %w", err) + } + if err := iter.Close(); err != nil { + return err + } + if !cleared { + return nil + } + // The final page is non-empty by construction (pages flush before + // append, so the last append leaves at least one entry) and its + // synced commit hardens the whole strip. + return flush(pebble.Sync) + }) +} + +// PutSourceCacheEntry writes the manifest entry for (rowKind, scopeKey). +// Zero-row scopes still get entries — the validator must survive to the +// next sync even when the scope produced no rows. +func (e *Engine) PutSourceCacheEntry(ctx context.Context, rowKind, scopeKey, cacheValidator string) error { + // Reject unknown kinds at the write: the seal pass hard-errors on any + // manifest entry whose kind it cannot count, manifest entries are + // individually undeletable, and EndSync retries fail identically — an + // unvalidated kind would turn a caller typo into an unsealable + // artifact. + if _, ok := sourceCacheRowKindSpecs[rowKind]; !ok { + return fmt.Errorf("source cache manifest: unknown row kind %q", rowKind) + } + if cacheValidator == "" { + return errors.New("source cache manifest: cache validator is required") + } + return e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + rec := &v3.SourceCacheEntryRecord{} + rec.SetRowKind(rowKind) + rec.SetScopeKey(scopeKey) + rec.SetCacheValidator(cacheValidator) + rec.SetDiscoveredAt(timestamppb.Now()) + val, err := marshalRecord(rec) + if err != nil { + return err + } + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + if e.test.sourceCacheManifestWriteHook != nil { + if err := e.test.sourceCacheManifestWriteHook(); err != nil { + return err + } + } + return e.db.SourceCacheSet(encodeSourceCacheEntryKey(rowKind, scopeKey), val, opts) + }) +} + +// SourceCachePoisoned reports whether (rowKind, scopeKey) carries a +// poison marker (CO-015): a mutation in this store removed a row from +// the scope's stamped set on behalf of something other than the scope +// itself — a cross-scope restamp or an out-of-scope delete. A poisoned +// scope must be treated as a lookup miss by orchestration and is +// refused as a replay source by preflight; it re-fetches cold and the +// next sync's fresh artifact starts unpoisoned. +func (e *Engine) SourceCachePoisoned(ctx context.Context, rowKind, scopeKey string) (bool, error) { + _, closer, err := e.db.Get(rawdb.SourceCachePoisonKey(rowKind, scopeKey)) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return false, nil + } + return false, err + } + _ = closer.Close() + return true, nil +} + +// GetSourceCacheEntry returns the manifest entry for (rowKind, scopeKey), +// or pebble.ErrNotFound. +func (e *Engine) GetSourceCacheEntry(ctx context.Context, rowKind, scopeKey string) (*v3.SourceCacheEntryRecord, error) { + val, closer, err := e.db.Get(encodeSourceCacheEntryKey(rowKind, scopeKey)) + if err != nil { + return nil, err + } + defer closer.Close() + rec := &v3.SourceCacheEntryRecord{} + if err := unmarshalRecord(val, rec); err != nil { + return nil, fmt.Errorf("GetSourceCacheEntry: unmarshal: %w", err) + } + return rec, nil +} + +// InvalidateSourceCacheReplayState removes upstream validators from a +// compaction output. Rebuild compactions also drop the source-scope indexes +// they synthesized while materializing winner rows; fold keeps those indexes +// because rebuilding or deleting them record-by-record would defeat fold's +// bounded-write design. Range tombstones make both forms O(1) in row count. +func (e *Engine) InvalidateSourceCacheReplayState(ctx context.Context, dropScopeIndexes bool) error { + return e.withWriteAllowSealed(func() error { + if err := ctx.Err(); err != nil { + return err + } + batch := e.db.NewRecordBatch() + defer batch.Close() + if err := batch.StageSourceCacheReplayInvalidation(dropScopeIndexes); err != nil { + return err + } + // Compaction does not publish the artifact until Close checkpoints and + // fsyncs the engine. Match the fold batches' NoSync policy so this + // constant-size tombstone does not add a standalone fsync. + return batch.Commit(pebble.NoSync) + }) +} + +// DeleteGrantRecordBounded deletes a grant by canonical public id WITHOUT +// the O(all grants) stored-external-id scan fallback that the interactive +// DeleteGrantRecord path is allowed to take. Used by the source-cache +// tombstone path, where a mass-removal round would otherwise pay a full +// keyspace scan PER already-absent id. +// +// Consequence, by design: a grant stored under a connector-CUSTOM id (one +// that isn't the SDK concat shape) is unreachable here and the delete +// no-ops. Connectors with custom grant ids must use principal-scoped +// tombstones (SourceCacheRecord.deleted_principal_ids) instead — documented +// in the annotation proto. +// +// Acts unscoped: deleting a stamped row through this path poisons the +// row's scope (CO-015). Scope-acting tombstones use +// DeleteGrantRecordsBounded with an acting scope instead. +func (e *Engine) DeleteGrantRecordBounded(ctx context.Context, externalID string) error { + return e.withWrite(func() error { + id, err := e.resolveGrantIdentityByCandidates(ctx, externalID) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return nil // absent (or custom-id) — tombstone no-op + } + return err + } + return e.deleteGrantByIdentityLocked(id) + }) +} + +// DeleteGrantRecordsBounded validates every canonical public id before staging +// any tombstone, then commits the resolved deletes in bounded chunks (same +// page mechanics as the scoped tombstone paths; deletion is idempotent, so an +// error mid-way retries convergently). Resolution keeps +// DeleteGrantRecordBounded's candidate-only contract: missing or +// connector-custom ids are no-ops, while an ambiguous id rejects the entire +// request. actingScope is the scope on whose behalf the tombstones act: +// deleting that scope's own rows stages no poison, while deleting a row +// stamped with any OTHER scope poisons it (CO-015). +func (e *Engine) DeleteGrantRecordsBounded(ctx context.Context, externalIDs []string, actingScope string) error { + return e.withWrite(func() error { + identities := make([]grantIdentity, 0, len(externalIDs)) + seen := make(map[grantIdentity]struct{}, len(externalIDs)) + for _, externalID := range externalIDs { + if err := ctx.Err(); err != nil { + return err + } + id, err := e.resolveGrantIdentityByCandidates(ctx, externalID) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + continue + } + return err + } + if _, ok := seen[id]; ok { + continue + } + seen[id] = struct{}{} + identities = append(identities, id) + } + if len(identities) == 0 { + return nil + } + + deletes := newSourceCacheDeleteBatch(e, "grants-canonical", actingScope, writeOpts(e.opts.durability)) + defer deletes.close() + for _, id := range identities { + key := encodeGrantIdentityKey(id) + oldVal, closer, err := e.db.Get(key) + if errors.Is(err, pebble.ErrNotFound) { + continue + } + if err != nil { + return err + } + if err := deletes.batch.StageGrantDelete(key, oldVal); err != nil { + _ = closer.Close() + return err + } + _ = closer.Close() + if err := deletes.staged(true); err != nil { + return err + } + } + return deletes.commit(true) + }) +} + +// DeleteResourceRecordsBounded deletes resources by (resource_type_id, +// resource_id) in bounded chunks, acting for actingScope — the resources +// analog of DeleteGrantRecordsBounded, replacing a per-id single-commit +// loop for the canonical tombstone path. Absent ids are no-ops. +func (e *Engine) DeleteResourceRecordsBounded(ctx context.Context, refs []ResourceRef, actingScope string) error { + if len(refs) == 0 { + return nil + } + return e.withWrite(func() error { + deletes := newSourceCacheDeleteBatch(e, "resources-canonical", actingScope, writeOpts(e.opts.durability)) + defer deletes.close() + for _, ref := range refs { + if err := ctx.Err(); err != nil { + return err + } + key := encodeResourceKey(ref.ResourceTypeID, ref.ResourceID) + oldVal, closer, err := e.db.Get(key) + if errors.Is(err, pebble.ErrNotFound) { + continue + } + if err != nil { + return err + } + if err := deletes.batch.StageResourceDelete(key, oldVal, ref.ResourceTypeID, ref.ResourceID); err != nil { + _ = closer.Close() + return err + } + _ = closer.Close() + if err := deletes.staged(true); err != nil { + return err + } + } + return deletes.commit(true) + }) +} + +// ResourceRef addresses one resource row for the bounded tombstone path. +type ResourceRef struct { + ResourceTypeID string + ResourceID string +} + +type sourceCacheDeleteBatch struct { + engine *Engine + batch *rawdb.RecordBatch + opts *pebble.WriteOptions + kind string + // actingScope is the scope on whose behalf the deletes act; applied + // to every minted batch (chunked commits re-mint) so a scope's own + // tombstones never stage poison against it (CO-015). + actingScope string + // onCommit runs after every commit that landed at least one delete + // (intermediate chunks and the final one). Engine-state invalidation + // keyed on keyspace mutation (the bare-id entitlement lookup) must + // fire per landed chunk, not on function exit: readers of that state + // synchronize on their own mutex, not the write barrier, so a bump + // deferred to the end of a long id loop leaves a window where a + // concurrent lookup serves rows a chunk already deleted. + onCommit func() + limit int + operations int + pendingDeleted int64 + committedDeleted int64 +} + +func newSourceCacheDeleteBatch(e *Engine, kind, actingScope string, opts *pebble.WriteOptions) *sourceCacheDeleteBatch { + limit := replayBatchRows + if e.test.sourceCacheDeleteBatchRows > 0 { + limit = e.test.sourceCacheDeleteBatchRows + } + b := &sourceCacheDeleteBatch{ + engine: e, + batch: e.db.NewRecordBatch(), + opts: opts, + kind: kind, + actingScope: actingScope, + limit: limit, + } + b.batch.SetActingSourceScope(actingScope) + return b +} + +func (b *sourceCacheDeleteBatch) staged(rowDeleted bool) error { + b.operations++ + if rowDeleted { + b.pendingDeleted++ + } + if b.operations < b.limit { + return nil + } + return b.commit(false) +} + +func (b *sourceCacheDeleteBatch) commit(final bool) error { + if b.operations == 0 { + return nil + } + if b.engine.test.sourceCacheDeleteCommitHook != nil { + if err := b.engine.test.sourceCacheDeleteCommitHook(b.kind, b.operations, final); err != nil { + return err + } + } + if err := b.batch.Commit(b.opts); err != nil { + return err + } + landed := b.pendingDeleted + b.committedDeleted += landed + _ = b.batch.Close() + b.batch = nil + b.operations = 0 + b.pendingDeleted = 0 + if !final { + b.batch = b.engine.db.NewRecordBatch() + b.batch.SetActingSourceScope(b.actingScope) + } + if landed > 0 && b.onCommit != nil { + b.onCommit() + } + return nil +} + +func (b *sourceCacheDeleteBatch) close() { + if b.batch == nil { + return + } + _ = b.batch.Close() + b.batch = nil +} + +// DeleteGrantsByPrincipalsInScope deletes every grant row in the CURRENT +// store stamped with scopeKey whose principal id is in principalIDs — +// the engine side of principal-scoped delta tombstones +// (SourceCacheRecord.deleted_principal_ids). +// +// One prefix scan of the scope's by_source_scope index resolves +// everything: the index tail IS the grant identity, so the primary key +// and every secondary index key for a match are constructible from the +// index key alone — no value reads, no string resolution, no guessing. +// A principal with no rows in the scope is a no-op (providers tombstone +// objects the client never synced). Deleting a missing secondary index +// entry is a pebble no-op, which covers the mixed inline/deferred +// by_principal state mid-sync. +// +// Complexity: O(scope size) tuple-walks per call regardless of tombstone +// count — callers batch a page's tombstones into one call. Deletes commit in +// bounded chunks; an error returns the count from chunks that already landed, +// and retry converges because deletion is idempotent. +func (e *Engine) DeleteGrantsByPrincipalsInScope(ctx context.Context, scopeKey string, principalIDs map[string]struct{}) (int64, error) { + if len(principalIDs) == 0 { + return 0, nil + } + prefix := encodeGrantBySourceScopePrefix(scopeKey) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + deletes := newSourceCacheDeleteBatch(e, "grant-principals", scopeKey, opts) + defer deletes.close() + defer func() { deleted = deletes.committedDeleted }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + // Tail layout: ent_rt | ent_rid | flag | ent_tail | prin_rt | prin_id + // (identical to the grant primary key tail; decoder shared with + // the primary-prefix scan paths in grants.go). + id, ok := decodeGrantIdentityTail(key, prefix) + if !ok { + continue // malformed index key — defensive skip + } + if _, hit := principalIDs[id.principalID]; !hit { + continue + } + // Primary key = grant header + the identity tail verbatim. + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeGrant) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + oldVal, closer, getErr := e.db.Get(priKey) + if errors.Is(getErr, pebble.ErrNotFound) { + if err := deletes.batch.StageSourceScopeOrphanIndexDelete(key); err != nil { + return err + } + if err := deletes.staged(false); err != nil { + return err + } + continue + } + if getErr != nil { + return getErr + } + if err := deletes.batch.StageGrantDelete(priKey, oldVal); err != nil { + closer.Close() + return err + } + closer.Close() + if err := deletes.staged(true); err != nil { + return err + } + } + if err := iter.Error(); err != nil { + return err + } + return deletes.commit(true) + }) + if err != nil { + return deleted, err + } + return deleted, nil +} + +// DeleteGrantsByExternalIDsInScope deletes every grant row in the CURRENT +// store stamped with scopeKey whose STORED grant id (external id, which +// may be a connector-custom shape) is in ids. One scan of the scope's +// index, loading each candidate's primary row to compare the stored id — +// bounded by the scope's row count, never the whole keyspace. This is the +// tombstone path for connectors with custom grant ids whose scopes span +// multiple resources (so principal-scoped deletes would over-delete). Deletes +// commit in bounded chunks and report committed progress on error. +func (e *Engine) DeleteGrantsByExternalIDsInScope(ctx context.Context, scopeKey string, ids map[string]struct{}) (int64, error) { + if len(ids) == 0 { + return 0, nil + } + prefix := encodeGrantBySourceScopePrefix(scopeKey) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + deletes := newSourceCacheDeleteBatch(e, "grant-external-ids", scopeKey, opts) + defer deletes.close() + defer func() { deleted = deletes.committedDeleted }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + _, ok := decodeGrantIdentityTail(key, prefix) + if !ok { + continue // malformed index key — defensive skip + } + // Primary key = grant header + the identity tail verbatim. + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeGrant) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + + val, closer, err := e.db.Get(priKey) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + if err := deletes.batch.StageSourceScopeOrphanIndexDelete(key); err != nil { + return err + } + if err := deletes.staged(false); err != nil { + return err + } + continue + } + return err + } + rec := &v3.GrantRecord{} + uerr := unmarshalRecord(val, rec) + if uerr != nil { + closer.Close() + return fmt.Errorf("DeleteGrantsByExternalIDsInScope: unmarshal: %w", uerr) + } + if _, hit := ids[rec.GetExternalId()]; !hit { + closer.Close() + continue + } + if err := deletes.batch.StageGrantDelete(priKey, val); err != nil { + closer.Close() + return err + } + closer.Close() + if err := deletes.staged(true); err != nil { + return err + } + } + if err := iter.Error(); err != nil { + return err + } + return deletes.commit(true) + }) + if err != nil { + return deleted, err + } + return deleted, nil +} + +// DeleteResourcesByIDsInScope deletes every resource row in the CURRENT +// store stamped with scopeKey whose resource id is in resourceIDs (any +// resource type) — principal-scoped tombstones for RowKindResources. Deletes +// commit in bounded chunks and report committed progress on error. +func (e *Engine) DeleteResourcesByIDsInScope(ctx context.Context, scopeKey string, resourceIDs map[string]struct{}) (int64, error) { + if len(resourceIDs) == 0 { + return 0, nil + } + prefix := encodeResourceBySourceScopePrefix(scopeKey) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + deletes := newSourceCacheDeleteBatch(e, "resources", scopeKey, opts) + defer deletes.close() + defer func() { deleted = deletes.committedDeleted }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + // Tail layout: resource_type_id | resource_id. + rtBytes, next, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok || next >= len(tail) { + continue + } + ridBytes, _, ok := codec.DecodeTupleStringAlias(tail, next+1) + if !ok { + continue + } + if _, hit := resourceIDs[string(ridBytes)]; !hit { + continue + } + rt, rid := string(rtBytes), string(ridBytes) + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeResource) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + rowDeleted := false + if val, closer, getErr := e.db.Get(priKey); getErr == nil { + err := deletes.batch.StageResourceDelete(priKey, val, rt, rid) + closer.Close() + if err != nil { + return err + } + rowDeleted = true + } else if errors.Is(getErr, pebble.ErrNotFound) { + if err := deletes.batch.StageSourceScopeOrphanIndexDelete(key); err != nil { + return err + } + } else { + return getErr + } + if err := deletes.staged(rowDeleted); err != nil { + return err + } + } + if err := iter.Error(); err != nil { + return err + } + return deletes.commit(true) + }) + if err != nil { + return deleted, err + } + return deleted, nil +} + +// grantValueHasSourcesRaw reports whether a marshaled GrantRecord carries +// at least one sources entry (field 9), without unmarshaling. +func grantValueHasSourcesRaw(value []byte) (bool, error) { + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return false, protowire.ParseError(n) + } + value = value[n:] + if num == 9 { + return true, nil + } + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return false, protowire.ParseError(n) + } + value = value[n:] + } + return false, nil +} + +// stripExpanderSourcesRaw clears a replayed grant's Sources map when it is +// expander-written, so the current sync's expansion recomputes it from +// true state instead of inheriting contributions that may have been +// removed upstream. Classification mirrors RollbackExpansion: a Sources +// map containing a self-source entry (keyed by the grant's own entitlement +// id) was written by the expander; one without a self-source is +// connector-set public data and is preserved. Returns (newValue, true) when +// the record was rewritten, (nil, false) when the original bytes should be +// copied verbatim. +func stripExpanderSourcesRaw(value []byte, ownEntitlementID string) ([]byte, bool, error) { + r := &v3.GrantRecord{} + if err := unmarshalRecord(value, r); err != nil { + return nil, false, fmt.Errorf("source cache replay: unmarshal grant for sources strip: %w", err) + } + sources := r.GetSources() + if len(sources) == 0 { + return nil, false, nil + } + if _, hasSelf := sources[ownEntitlementID]; !hasSelf { + // No self-source: connector-set Sources. Preserve verbatim. + return nil, false, nil + } + r.SetSources(nil) + stripped, err := marshalRecord(r) + if err != nil { + return nil, false, fmt.Errorf("source cache replay: re-marshal grant after sources strip: %w", err) + } + return stripped, true, nil +} + +// decodeResourcePrimaryTail decodes (resource_type_id, resource_id) +// from a resource primary key (v3 | typeResource | 0x00 | rt | 0x00 | rid). +func decodeResourcePrimaryTail(priKey []byte) (string, string, error) { + const headerLen = 3 // versionV3, typeResource, separator + if len(priKey) <= headerLen { + return "", "", fmt.Errorf("source cache replay: malformed resource primary key %x", priKey) + } + tail := priKey[headerLen:] + rtBytes, next, err := codec.DecodeTupleStringTo(nil, tail, 0) + if err != nil { + return "", "", err + } + if next >= len(tail) { + return "", "", fmt.Errorf("source cache replay: resource primary key missing resource_id: %x", priKey) + } + ridBytes, _, err := codec.DecodeTupleStringTo(nil, tail, next+1) + if err != nil { + return "", "", err + } + return string(rtBytes), string(ridBytes), nil +} + +// replayPrimaryFromIndexKey derives a record's primary key from its +// by_source_scope index key. The index prefix (header|0x00|scope|0x00) +// is followed by exactly the identity tuple that forms the primary +// key's tail, so the primary is header' + 0x00-separated remainder. +func replayPrimaryFromIndexKey(indexKey, indexPrefix []byte, primaryHeader [2]byte) ([]byte, error) { + if len(indexKey) <= len(indexPrefix) { + return nil, fmt.Errorf("source cache replay: malformed index key %x", indexKey) + } + tail := indexKey[len(indexPrefix):] + key := make([]byte, 0, 3+len(tail)) + key = append(key, primaryHeader[0], primaryHeader[1], 0x00) + return append(key, tail...), nil +} + +// validateReplaySourceScope proves the target scope's primary↔index +// biconditional before the destination is mutated, in O(scope size): +// +// 1. Walk the scope's by_source_scope index. Every entry must resolve +// to a primary whose value stamp names this scope (no orphans, no +// stale entries). Index keys are unique and each derives a distinct +// primary, so the walk's cardinality is |stamped primaries that are +// indexed|. +// 2. Compare that cardinality against the manifest's sealed row_count — +// the number of primaries stamped with this scope, counted from the +// primary keyspace at EndSync (sealSourceCacheRowCounts, CO-004). +// Equality makes the index→primary injection surjective: every +// stamped primary is indexed. A stamped primary missing its index +// entry shows up as a count shortfall, which is exactly what the +// deleted O(all primaries) scan used to detect row-by-row. +// +// The equivalence holds because the source is immutable between seal +// and replay: a published c1z is opened read-only, and the sealed +// engine admits no record mutations between counting and the ended_at +// stamp. A manifest entry WITHOUT a sealed count is a hard error — +// replay-eligible artifacts are sealed by an SDK that counts, so +// absence means the seal step was bypassed, not an older format. +// +// Note what the count deliberately does NOT vouch for: mid-sync +// partition damage that happened BEFORE the seal (a cross-scope +// restamp, or an unscoped delete such as external-principal +// reconciliation removing a stamped row) leaves index, stamps, and +// count self-consistent with the damaged set. That is the per-scope +// poison marker's job, staged durably at mutation time — not the +// count's. +func validateReplaySourceScope( + ctx context.Context, + prev *Engine, + rowKind string, + recordType byte, + scopeField protowire.Number, + scopeKey string, + indexPrefix []byte, +) error { + if err := ctx.Err(); err != nil { + return err + } + entry, err := prev.GetSourceCacheEntry(ctx, rowKind, scopeKey) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return fmt.Errorf("source cache replay: preflight %s scope %q has no manifest entry", rowKind, scopeKey) + } + return fmt.Errorf("source cache replay: preflight %s scope %q manifest read: %w", rowKind, scopeKey, err) + } + if entry.GetInvalidated() { + return fmt.Errorf("source cache replay: preflight %s scope %q manifest entry is invalidated", rowKind, scopeKey) + } + poisonedScope, err := prev.SourceCachePoisoned(ctx, rowKind, scopeKey) + if err != nil { + return fmt.Errorf("source cache replay: preflight %s scope %q poison read: %w", rowKind, scopeKey, err) + } + if poisonedScope { + return fmt.Errorf( + "source cache replay: preflight %s scope %q is poisoned: the source sync observed a row-partition violation "+ + "(cross-scope restamp or out-of-scope delete) against this scope; it must re-fetch cold", + rowKind, scopeKey, + ) + } + if !entry.HasRowCount() { + return fmt.Errorf( + "source cache replay: preflight %s scope %q manifest entry has no sealed row count (source was not sealed by a counting EndSync)", + rowKind, scopeKey, + ) + } + want := entry.GetRowCount() + + indexes, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: indexPrefix, + UpperBound: upperBoundOf(indexPrefix), + }) + if err != nil { + return fmt.Errorf("source cache replay: preflight %s indexes: %w", rowKind, err) + } + defer indexes.Close() + primaryHeader := [2]byte{versionV3, recordType} + var got uint64 + var scanned int + for indexes.First(); indexes.Valid(); indexes.Next() { + scanned++ + if scanned&0x3FF == 0 { + if err := ctx.Err(); err != nil { + return err + } + } + primaryKey, err := replayPrimaryFromIndexKey(indexes.Key(), indexPrefix, primaryHeader) + if err != nil { + return err + } + value, closer, err := prev.db.Get(primaryKey) + if err != nil { + return fmt.Errorf("source cache replay: preflight %s index %x has no primary: %w", rowKind, indexes.Key(), err) + } + stamp, scanErr := rawdb.ScanSourceScopeKeyRaw(value, scopeField) + closer.Close() + if scanErr != nil { + return fmt.Errorf("source cache replay: preflight %s indexed primary %x: %w", rowKind, primaryKey, scanErr) + } + if stamp != scopeKey { + return fmt.Errorf( + "source cache replay: preflight %s index scope %q resolves to primary stamped %q", + rowKind, + scopeKey, + stamp, + ) + } + got++ + } + if err := indexes.Error(); err != nil { + return fmt.Errorf("source cache replay: preflight %s indexes: %w", rowKind, err) + } + if got != want { + return fmt.Errorf( + "source cache replay: preflight %s scope %q index cardinality %d does not match sealed row count %d (stamped primary missing its index entry, or post-seal mutation)", + rowKind, scopeKey, got, want, + ) + } + return nil +} + +// clearReplayDestinationScopeLocked gives pure replay replacement semantics: +// the destination target partition is removed before the source partition is +// copied. The caller holds the engine write barrier. Deletes are committed in +// the same bounded row batches as replay, so interruption followed by retry +// converges without retaining destination-only rows. The deletes act FOR the +// scope being replayed (SetActingSourceScope): replacing your own partition +// is the replay flow itself, never a poison event. +func (e *Engine) clearReplayDestinationScopeLocked( + ctx context.Context, + rowKind string, + recordType byte, + scopeKey string, + indexPrefix []byte, + opts *pebble.WriteOptions, +) (int, error) { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: indexPrefix, + UpperBound: upperBoundOf(indexPrefix), + }) + if err != nil { + return 0, err + } + defer iter.Close() + primaryHeader := [2]byte{versionV3, recordType} + batch := e.db.NewRecordBatch() + batch.SetActingSourceScope(scopeKey) + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + deletedInBatch := 0 + deleted := 0 + commit := func(final bool) error { + if rowsInBatch == 0 { + return nil + } + if e.test.sourceCacheReplayClearCommitHook != nil { + if err := e.test.sourceCacheReplayClearCommitHook(rowKind, rowsInBatch, final); err != nil { + return err + } + } + if err := batch.Commit(opts); err != nil { + return err + } + deleted += deletedInBatch + // Same per-chunk contract as sourceCacheDeleteBatch.onCommit: + // bare-id lookups synchronize on entIDLookupMu only, so the + // entitlement keyspace must invalidate the cached map as each + // chunk lands, not when the caller finishes. + if deletedInBatch > 0 && recordType == typeEntitlement { + e.noteEntitlementKeyspaceWrite() + } + _ = batch.Close() + batch = e.db.NewRecordBatch() + batch.SetActingSourceScope(scopeKey) + rowsInBatch = 0 + deletedInBatch = 0 + return nil + } + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return deleted, err + } + indexKey := append([]byte(nil), iter.Key()...) + primaryKey, err := replayPrimaryFromIndexKey(indexKey, indexPrefix, primaryHeader) + if err != nil { + return deleted, err + } + value, closer, err := e.db.Get(primaryKey) + switch { + case errors.Is(err, pebble.ErrNotFound): + if err := batch.StageSourceScopeOrphanIndexDelete(indexKey); err != nil { + return deleted, err + } + case err != nil: + return deleted, err + default: + switch recordType { + case typeGrant: + err = batch.StageGrantDelete(primaryKey, value) + case typeEntitlement: + err = batch.StageEntitlementDelete(primaryKey, value) + case typeResource: + var resourceTypeID, resourceID string + resourceTypeID, resourceID, err = decodeResourcePrimaryTail(primaryKey) + if err == nil { + err = batch.StageResourceDelete(primaryKey, value, resourceTypeID, resourceID) + } + default: + err = fmt.Errorf("source cache replay: clear destination: invalid row kind %q", rowKind) + } + closer.Close() + if err != nil { + return deleted, err + } + deletedInBatch++ + } + rowsInBatch++ + if rowsInBatch >= e.sourceCacheReplayBatchLimit() { + if err := commit(false); err != nil { + return deleted, err + } + } + } + if err := iter.Error(); err != nil { + return deleted, err + } + if err := commit(true); err != nil { + return deleted, err + } + return deleted, nil +} + +// ReplaySourceCacheGrants copies every grant stamped with scopeKey from +// prev into the receiver: raw primary copy plus index synthesis from the +// raw value (principal, needs_expansion, source-scope families). Mirrors +// PutGrantRecords' read-before-write index cleanup when the receiver +// already holds a record at the same identity. +func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scopeKey string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeGrantBySourceScopePrefix(scopeKey) + primaryHeader := [2]byte{versionV3, typeGrant} + if err := validateReplaySourceScope(ctx, prev, "grants", typeGrant, 10, scopeKey, prefix); err != nil { + return SourceCacheReplayResult{}, err + } + + committedRows := 0 + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + deleted, err := e.clearReplayDestinationScopeLocked(ctx, "grants", typeGrant, scopeKey, prefix, opts) + if deleted > 0 { + _ = e.takeFreshGrantsEmpty() + } + if err != nil { + return err + } + batch := e.db.NewRecordBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + // Consumed in a defer keyed on rows whose commit LANDED, not on + // success: a replay that fails after an intermediate commit + // (large scope, error or cancellation mid-copy) has already + // populated the keyspace, and while the failing action unwinds, + // concurrently draining workers can still write — a first + // PutGrantRecords taking the empty-keyspace fast path over + // partially replayed identities would skip index cleanup. + // (committedRows itself is function-scoped so the error return + // can report landed progress.) + defer func() { + if committedRows > 0 { + _ = e.takeFreshGrantsEmpty() + } + }() + + sourceRow := 0 + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + if e.test.sourceCacheReplayReadHook != nil { + if err := e.test.sourceCacheReplayReadHook("grants", sourceRow); err != nil { + return err + } + } + sourceRow++ + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + // An orphan index entry (no primary) was proven absent by + // validateReplaySourceScope over this same range on the + // same immutable source, so ErrNotFound here means the + // preflight and the copy loop disagree — a bug in one of + // them, never a legitimate source state. Fail loudly + // rather than skip: silently dropping a row the index + // promised is the silent-data-loss shape this subsystem + // exists to refuse. + return fmt.Errorf("source cache replay: get prev grant (index entry %x passed preflight): %w", iter.Key(), getErr) + } + + _, _, entID, _, _, needsExpansion, scanErr := scanGrantIndexFieldsRaw(val) + if scanErr != nil { + closer.Close() + return scanErr + } + srcScope, scanErr := rawdb.ScanSourceScopeKeyRaw(val, 10) + if scanErr != nil { + closer.Close() + return scanErr + } + // Stamp re-check on the same preflight-proven invariant: an + // index entry resolving to a row stamped for a different scope + // would inject rows upstream never returned for this scope, so + // disagreement with the preflight is a hard error, not a skip. + if srcScope != scopeKey { + closer.Close() + return fmt.Errorf( + "source cache replay: grants index scope %q resolves to primary stamped %q after passing preflight", + scopeKey, srcScope, + ) + } + + // Replay-equivalence: a cached sync must reproduce what a full + // resync would produce. The one field where a verbatim copy + // diverges is expander-written Sources — the previous sync's + // expansion baked contributions into direct grants, and + // re-expansion only ADDS, so a contribution removed this sync + // (via a delta tombstone or a refetched page) would survive + // forever. Strip expander-written Sources so the current sync's + // expansion recomputes them from true state; connector-set + // Sources (no self-source entry — same classification as + // RollbackExpansion) are connector data and are preserved + // verbatim. The probe is a cheap protowire scan; the vast + // majority of rows carry no Sources and stay on the raw-copy + // path. + writeVal := val + if hasSources, probeErr := grantValueHasSourcesRaw(val); probeErr != nil { + closer.Close() + return probeErr + } else if hasSources { + stripped, strippedOK, stripErr := stripExpanderSourcesRaw(val, entID) + if stripErr != nil { + closer.Close() + return stripErr + } + if strippedOK { + writeVal = stripped + } + } + var oldVal []byte + var oldCloser io.Closer + currentVal, currentCloser, oldErr := e.db.Get(priKey) + if oldErr == nil { + oldVal, oldCloser = currentVal, currentCloser + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current grant: %w", oldErr) + } + if err := batch.StageGrantPutInline(priKey, writeVal, oldVal, needsExpansion); err != nil { + if oldCloser != nil { + _ = oldCloser.Close() + } + closer.Close() + return err + } + if oldCloser != nil { + _ = oldCloser.Close() + } + closer.Close() + if needsExpansion { + res.NeedsExpansion = true + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= e.sourceCacheReplayBatchLimit() { + if e.test.sourceCacheReplayCommitHook != nil { + if err := e.test.sourceCacheReplayCommitHook("grants", rowsInBatch, false); err != nil { + return err + } + } + if err := batch.Commit(opts); err != nil { + return err + } + committedRows += rowsInBatch + _ = batch.Close() + batch = e.db.NewRecordBatch() + rowsInBatch = 0 + } + } + if err := e.sourceCacheReplayIteratorError("grants", iter); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + if e.test.sourceCacheReplayCommitHook != nil { + if err := e.test.sourceCacheReplayCommitHook("grants", rowsInBatch, true); err != nil { + return err + } + } + if err := batch.Commit(opts); err != nil { + return err + } + // Replay populated the fresh sync's grant keyspace directly. The + // first overlay PutGrantRecords must therefore perform its normal + // read-before-write index cleanup, rather than claiming the + // keyspace is still empty and leaving replayed index entries stale + // (consumed by the defer above). + committedRows += rowsInBatch + return nil + }) + if err != nil { + // Bounded intermediate batches may already have landed (that is + // the retry seam), so report committed progress with the error, + // matching the scoped-delete siblings. Rows counts only rows + // whose commit landed; the failing batch's staged rows are + // dropped. NeedsExpansion accumulates at STAGE time and may + // overreport a row that never committed — the safe direction + // (arming expansion is idempotent and add-only), where + // underreporting could leave a committed expandable grant + // unexpanded. + res.Rows = int64(committedRows) + return res, err + } + return res, nil +} + +// ReplaySourceCacheEntitlements copies every entitlement stamped with +// scopeKey from prev into the receiver. +func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine, scopeKey string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeEntitlementBySourceScopePrefix(scopeKey) + primaryHeader := [2]byte{versionV3, typeEntitlement} + if err := validateReplaySourceScope(ctx, prev, "entitlements", typeEntitlement, 11, scopeKey, prefix); err != nil { + return SourceCacheReplayResult{}, err + } + + committedRows := 0 + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + deleted, err := e.clearReplayDestinationScopeLocked(ctx, "entitlements", typeEntitlement, scopeKey, prefix, opts) + if deleted > 0 { + e.noteEntitlementKeyspaceWrite() + _ = e.takeFreshEntitlementsEmpty() + } + if err != nil { + return err + } + batch := e.db.NewRecordBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + // Keyed on rows whose commit LANDED, not on success (see the + // grants replay): a partial replay has already mutated the + // entitlement keyspace, so the bare-id lookup map must be + // invalidated and the empty-keyspace fast path disarmed even when + // the replay itself fails — draining workers can still resolve + // tombstones and write entitlements while the failure unwinds. + // (committedRows itself is function-scoped so the error return + // can report landed progress.) + defer func() { + if committedRows > 0 { + e.noteEntitlementKeyspaceWrite() + _ = e.takeFreshEntitlementsEmpty() + } + }() + + sourceRow := 0 + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + if e.test.sourceCacheReplayReadHook != nil { + if err := e.test.sourceCacheReplayReadHook("entitlements", sourceRow); err != nil { + return err + } + } + sourceRow++ + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + // ErrNotFound means the copy loop disagrees with the + // preflight over the same immutable range — a bug, never a + // legitimate source state (see the grants replay). + return fmt.Errorf("source cache replay: get prev entitlement (index entry %x passed preflight): %w", iter.Key(), getErr) + } + // Stamp re-check on the preflight-proven invariant (see the + // grants replay for rationale): disagreement is a hard error. + prevScope, prevScanErr := rawdb.ScanSourceScopeKeyRaw(val, 11) + if prevScanErr != nil { + closer.Close() + return prevScanErr + } + if prevScope != scopeKey { + closer.Close() + return fmt.Errorf( + "source cache replay: entitlements index scope %q resolves to primary stamped %q after passing preflight", + scopeKey, prevScope, + ) + } + var oldVal []byte + var oldCloser io.Closer + currentVal, currentCloser, oldErr := e.db.Get(priKey) + if oldErr == nil { + oldVal, oldCloser = currentVal, currentCloser + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current entitlement: %w", oldErr) + } + if err := batch.StageEntitlementPut(priKey, val, oldVal); err != nil { + if oldCloser != nil { + _ = oldCloser.Close() + } + closer.Close() + return err + } + if oldCloser != nil { + _ = oldCloser.Close() + } + closer.Close() + res.Rows++ + rowsInBatch++ + if rowsInBatch >= e.sourceCacheReplayBatchLimit() { + if e.test.sourceCacheReplayCommitHook != nil { + if err := e.test.sourceCacheReplayCommitHook("entitlements", rowsInBatch, false); err != nil { + return err + } + } + if err := batch.Commit(opts); err != nil { + return err + } + committedRows += rowsInBatch + // Same per-chunk contract as the clear half above and + // sourceCacheDeleteBatch.onCommit: bare-id readers take + // only entIDLookupMu, so each landed chunk must + // invalidate the cached map — deferring to function exit + // would let a reader build and keep serving a map + // missing every entitlement this chunk just committed. + e.noteEntitlementKeyspaceWrite() + _ = batch.Close() + batch = e.db.NewRecordBatch() + rowsInBatch = 0 + } + } + if err := e.sourceCacheReplayIteratorError("entitlements", iter); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + if e.test.sourceCacheReplayCommitHook != nil { + if err := e.test.sourceCacheReplayCommitHook("entitlements", rowsInBatch, true); err != nil { + return err + } + } + if err := batch.Commit(opts); err != nil { + return err + } + // The defer above covers the FINAL chunk's lookup invalidation + // (see lookup.go — later tombstones would silently miss replayed + // rows otherwise) and disarms the empty-keyspace fast path for + // the first overlay PutEntitlementRecords; intermediate chunks + // bumped as they landed above. Bump timing is conservative in + // both directions: entitlementIdentitiesForExternalID loads the + // generation BEFORE taking entIDLookupMu, so a map build racing + // any bump records the older generation and the next lookup + // rebuilds — a bump can never launder partially-replayed state + // as fresh. + committedRows += rowsInBatch + return nil + }) + if err != nil { + // Committed progress rides the error — see the grants replay. + res.Rows = int64(committedRows) + return res, err + } + return res, nil +} + +// ReplaySourceCacheResources copies every resource stamped with scopeKey +// from prev into the receiver, synthesizing by_parent and by_source_scope +// index entries from the raw value. +func (e *Engine) ReplaySourceCacheResources(ctx context.Context, prev *Engine, scopeKey string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeResourceBySourceScopePrefix(scopeKey) + primaryHeader := [2]byte{versionV3, typeResource} + if err := validateReplaySourceScope(ctx, prev, "resources", typeResource, 12, scopeKey, prefix); err != nil { + return SourceCacheReplayResult{}, err + } + + committedRows := 0 + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + deleted, err := e.clearReplayDestinationScopeLocked(ctx, "resources", typeResource, scopeKey, prefix, opts) + if deleted > 0 { + _ = e.takeFreshResourcesEmpty() + } + if err != nil { + return err + } + batch := e.db.NewRecordBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + // Keyed on rows whose commit LANDED, not on success — see the + // grants replay for rationale. (committedRows itself is + // function-scoped so the error return can report landed progress.) + defer func() { + if committedRows > 0 { + _ = e.takeFreshResourcesEmpty() + } + }() + + sourceRow := 0 + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + if e.test.sourceCacheReplayReadHook != nil { + if err := e.test.sourceCacheReplayReadHook("resources", sourceRow); err != nil { + return err + } + } + sourceRow++ + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + // ErrNotFound means the copy loop disagrees with the + // preflight over the same immutable range — a bug, never a + // legitimate source state (see the grants replay). + return fmt.Errorf("source cache replay: get prev resource (index entry %x passed preflight): %w", iter.Key(), getErr) + } + + rt, rid, decodeErr := decodeResourcePrimaryTail(priKey) + if decodeErr != nil { + closer.Close() + return decodeErr + } + srcScope, scanErr := rawdb.ScanSourceScopeKeyRaw(val, 12) + if scanErr != nil { + closer.Close() + return scanErr + } + // Stamp re-check on the preflight-proven invariant (see the + // grants replay for rationale): disagreement is a hard error. + if srcScope != scopeKey { + closer.Close() + return fmt.Errorf( + "source cache replay: resources index scope %q resolves to primary stamped %q after passing preflight", + scopeKey, srcScope, + ) + } + var oldVal []byte + var oldCloser io.Closer + currentVal, currentCloser, oldErr := e.db.Get(priKey) + if oldErr == nil { + oldVal, oldCloser = currentVal, currentCloser + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current resource: %w", oldErr) + } + + if err := batch.StageResourcePut(priKey, val, oldVal, rt, rid); err != nil { + if oldCloser != nil { + _ = oldCloser.Close() + } + closer.Close() + return err + } + if oldCloser != nil { + _ = oldCloser.Close() + } + closer.Close() + res.Rows++ + rowsInBatch++ + if rowsInBatch >= e.sourceCacheReplayBatchLimit() { + if e.test.sourceCacheReplayCommitHook != nil { + if err := e.test.sourceCacheReplayCommitHook("resources", rowsInBatch, false); err != nil { + return err + } + } + if err := batch.Commit(opts); err != nil { + return err + } + committedRows += rowsInBatch + _ = batch.Close() + batch = e.db.NewRecordBatch() + rowsInBatch = 0 + } + } + if err := e.sourceCacheReplayIteratorError("resources", iter); err != nil { + return err + } + if err := ctx.Err(); err != nil { + return err + } + if e.test.sourceCacheReplayCommitHook != nil { + if err := e.test.sourceCacheReplayCommitHook("resources", rowsInBatch, true); err != nil { + return err + } + } + if err := batch.Commit(opts); err != nil { + return err + } + // See grant replay above: direct replay writes mean the first + // overlay PutResourceRecords must not use the empty-keyspace + // fast-path, or old by_parent/by_source_scope entries survive + // (consumed by the defer above). + committedRows += rowsInBatch + return nil + }) + if err != nil { + // Committed progress rides the error — see the grants replay. + res.Rows = int64(committedRows) + return res, err + } + return res, nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go index e87c58d6..2bf6ccad 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/sync_runs.go @@ -31,8 +31,8 @@ func (e *Engine) PutSyncRunRecord(ctx context.Context, r *v3.SyncRunRecord) erro } // AllowSealed: sync-run metadata is legitimately stamped on a finished // sync — ToPebble preserves the source ended_at, the sanitizer applies - // diff links / supports_diff, and the compactor renames the folded sync - // — all after EndSync sealed the engine. + // supports_diff, and the compactor renames the folded sync — all after + // EndSync sealed the engine. return e.withWriteAllowSealed(func() error { val, err := marshalRecord(r) if err != nil { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/test_seams.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/test_seams.go index 2bbe2382..cc65934e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/test_seams.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/test_seams.go @@ -49,6 +49,42 @@ type testSeams struct { // post-arm obligation and no hook. recordCommitHook func() error + // sourceCacheReplayCommitHook runs immediately before each bounded replay + // batch commit. It provides deterministic high-water telemetry and a + // per-chunk failure seam without changing the replay iterator. + sourceCacheReplayCommitHook func(kind string, rows int, final bool) error + sourceCacheReplayBatchRows int + + // sourceCacheReplayClearCommitHook runs immediately before each bounded + // destination-clear batch commit. Replacement clear and replay copy are + // distinct commit loops and therefore require distinct failure seams. + sourceCacheReplayClearCommitHook func(kind string, rows int, final bool) error + + // sourceCacheReplayReadHook runs before each source index row is consumed. + // It supplies deterministic source-iteration errors at exact row cuts. + sourceCacheReplayReadHook func(kind string, row int) error + // sourceCacheReplayIteratorErrorHook runs at the real Iterator.Error + // disposition after source iteration and before the final batch commit. + // It proves that an iterator terminal error cannot be swallowed or followed + // by publication of the final staged rows. + sourceCacheReplayIteratorErrorHook func(kind string) error + + // sourceCacheDeleteCommitHook runs before each bounded scoped-tombstone + // commit. sourceCacheDeleteBatchRows lowers the production batch limit so + // tests can exercise interrupted multi-batch retry without whale fixtures. + sourceCacheDeleteCommitHook func(kind string, rows int, final bool) error + sourceCacheDeleteBatchRows int + + // sourceCacheManifestWriteHook runs immediately before a manifest entry is + // committed, after the value has been constructed. + sourceCacheManifestWriteHook func() error + + // poisonLogSetCap overrides the poison-warning dedup-set bound (engine.go, + // production 4096) so a test can reach the suppression branch with a + // handful of scopes: past the bound, unseen scopes stop logging behind a + // single one-time notice while already-seen scopes still deduplicate. + poisonLogSetCap int + // endSyncStampHook, when non-nil, runs immediately before the // ended_at stamp's PutSyncRunRecord commit in endSyncFinalize — // the in-process analog of the stamp commit failing. The diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/translate_v2.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/translate_v2.go index b713de98..5d61af3c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/translate_v2.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/translate_v2.go @@ -341,15 +341,32 @@ func V2ResourceToV3(syncID string, r *v2.Resource) *v3.ResourceRecord { }.Build() } return v3.ResourceRecord_builder{ - ResourceTypeId: r.GetId().GetResourceType(), - ResourceId: r.GetId().GetResource(), - DisplayName: r.GetDisplayName(), - Description: r.GetDescription(), - Parent: parent, - Annotations: r.GetAnnotations(), - CreatedAt: r.GetCreatedAt(), - Profile: r.GetProfile(), - Status: v2StatusToV3(r.GetStatus()), + ResourceTypeId: r.GetId().GetResourceType(), + ResourceId: r.GetId().GetResource(), + DisplayName: r.GetDisplayName(), + Description: r.GetDescription(), + Parent: parent, + Annotations: r.GetAnnotations(), + CreatedAt: r.GetCreatedAt(), + Profile: r.GetProfile(), + Status: v2StatusToV3(r.GetStatus()), + IconAssetExternalId: v2AssetExternalID(r.GetIcon()), + }.Build() +} + +func v2AssetExternalID(a *v2.AssetRef) string { + if a == nil { + return "" + } + return a.GetId() +} + +func assetExternalIDToV2AssetRef(assetExternalID string) *v2.AssetRef { + if assetExternalID == "" { + return nil + } + return v2.AssetRef_builder{ + Id: assetExternalID, }.Build() } @@ -401,6 +418,7 @@ func V3ResourceToV2(r *v3.ResourceRecord) *v2.Resource { Profile: r.GetProfile(), Status: v3StatusToV2(r.GetStatus()), CreatedAt: r.GetCreatedAt(), + Icon: assetExternalIDToV2AssetRef(r.GetIconAssetExternalId()), }.Build() } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go index db95f26b..180176b9 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/engine_registry.go @@ -203,8 +203,11 @@ func storeOptionsFromC1ZOptions(options *c1zOptions) StoreOptions { MaxDecodedPayloadBytes: maxDecodedPayloadBytes, MaxDecoderMemoryBytes: maxDecoderMemoryBytes, } + // Defensive only: NewStore, the sole caller, overwrites Engine with + // the selected driver's engine on the next line. Real default + // selection lives in selectStoreDriver. if out.Engine == "" { - out.Engine = c1zstore.EngineSQLite + out.Engine = c1zstore.EnginePebble } out.Pragmas = make([]StorePragma, 0, len(options.pragmas)) for _, p := range options.pragmas { @@ -218,7 +221,7 @@ func storeOptionsFromC1ZOptions(options *c1zOptions) StoreOptions { // Dispatch policy (in order): // // 1. If the file doesn't exist or is empty, honor the caller's -// `WithEngine(...)` choice (defaulting to EngineSQLite when +// `WithEngine(...)` choice (defaulting to EnginePebble when // unset). The about-to-be-written file gets the requested format. // 2. If the file exists with content, dispatch by the on-disk magic // byte — v1 → SQLite, v3 → whatever engine name the manifest @@ -226,6 +229,11 @@ func storeOptionsFromC1ZOptions(options *c1zOptions) StoreOptions { // case because we can't re-encode an existing file at open time; // the on-disk format is authoritative. This preserves the // read-any-format semantics that pre-dates the engine option. +// Exception: an EXPLICIT WithEngine(EnginePebble) on a writable v1 +// file converts it to Pebble in place. The EnginePebble default +// never triggers that conversion — an engine-less open of an +// existing v1 file stays SQLite, so read-intent callers (diff, +// stats, provisioning) don't rewrite files as a side effect. // // When the caller's WithEngine disagrees with the on-disk format we // log a warning so the divergence is observable. Callers that want @@ -235,7 +243,7 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO l := ctxzap.Extract(ctx) requested := options.engine if requested == "" { - requested = c1zstore.EngineSQLite + requested = c1zstore.EnginePebble } stat, err := os.Stat(outputFilePath) // #nosec G703 -- c1z path is caller-controlled by API design. @@ -267,7 +275,9 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO switch format { case C1ZFormatV1: // Maybe error if the file is read-only? - if requested == c1zstore.EnginePebble && !options.readOnly { + // Only an explicit pebble request converts; the engine default + // (options.engine == "") must not rewrite existing v1 files. + if options.engine == c1zstore.EnginePebble && !options.readOnly { // Close our header-read handle before converting: the conversion // renames a temp file over outputFilePath, which fails on Windows // if any handle to the destination is still open. Nil out f so @@ -277,11 +287,11 @@ func selectStoreDriver(ctx context.Context, outputFilePath string, options *c1zO if closeErr != nil { return nil, closeErr } - l.Debug("converting existing v1 c1z to pebble", zap.String("output_file_path", outputFilePath)) + l.Info("converting existing v1 c1z to pebble", zap.String("output_file_path", outputFilePath)) if err := convertExistingV1C1ZFile(ctx, outputFilePath, pebbleOpenOptionsFromC1Z(options)); err != nil { return nil, fmt.Errorf("select-store-driver: convert existing v1 c1z to pebble: %w", err) } - l.Debug("converted existing v1 c1z to pebble", zap.String("output_file_path", outputFilePath)) + l.Info("converted existing v1 c1z to pebble", zap.String("output_file_path", outputFilePath)) return requireEngineDriver(c1zstore.EnginePebble) } fileEngine = c1zstore.EngineSQLite diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/manifest.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/manifest.go index 625249ae..71fbe4ec 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/manifest.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3/manifest.go @@ -3,6 +3,7 @@ package v3 import ( "errors" "fmt" + "sort" "google.golang.org/protobuf/proto" "google.golang.org/protobuf/reflect/protodesc" @@ -33,6 +34,16 @@ var ( // The closure invariant: for every file F in the result, every file F // imports is also in the result. Reader-side verification can detect // any missing import and return ErrManifestIncompleteDescriptors. +// +// The result is sorted by file path so the set — and therefore the +// marshaled manifest that embeds it — is byte-deterministic across +// calls. proto's Deterministic marshal option only sorts map entries; +// repeated-field order is part of the message value, so emitting the +// closure in Go map-iteration order would make every save produce +// different manifest bytes (and a different manifest_xxh64) for +// identical content, defeating any use of the manifest as a content +// identity. Readers do not depend on any particular order +// (VerifyDescriptorClosure is set-based). func BuildDescriptorClosure() (*descriptorpb.FileDescriptorSet, error) { // Collect all files whose package is c1.storage.v3 OR which any // such file transitively imports. @@ -56,11 +67,20 @@ func BuildDescriptorClosure() (*descriptorpb.FileDescriptorSet, error) { return true }) + // Sort by path (the map key, so unique — no ties) rather than + // ranging the map: Go randomizes map iteration per range statement, + // which would permute the repeated field on every call. + paths := make([]string, 0, len(seen)) + for p := range seen { + paths = append(paths, p) + } + sort.Strings(paths) + set := &descriptorpb.FileDescriptorSet{ File: make([]*descriptorpb.FileDescriptorProto, 0, len(seen)), } - for _, fd := range seen { - set.File = append(set.File, protodesc.ToFileDescriptorProto(fd)) + for _, p := range paths { + set.File = append(set.File, protodesc.ToFileDescriptorProto(seen[p])) } return set, nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go index c1dcc309..abf949cb 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/pebble_store.go @@ -35,10 +35,7 @@ var _ connectorstore.Writer = (*pebbleStore)(nil) // the source/destination store (pkg/c1zsanitize keeps those interfaces // unexported). The assertions below make a refactor that drops one of these // methods from either engine break the build here, rather than silently -// disarming the sanitizer's sync-graph-metadata preservation. -type sanitizeSyncLinkWriter interface { - SetSyncLink(ctx context.Context, syncID string, linkedSyncID string) error -} +// disarming the sanitizer's sync-run-metadata preservation. type sanitizeSupportsDiffWriter interface { SetSupportsDiff(ctx context.Context, syncID string) error } @@ -47,10 +44,8 @@ type sanitizeSyncRunMetadataReader interface { } var ( - _ sanitizeSyncLinkWriter = (*pebbleStore)(nil) _ sanitizeSupportsDiffWriter = (*pebbleStore)(nil) _ sanitizeSyncRunMetadataReader = (*pebbleStore)(nil) - _ sanitizeSyncLinkWriter = (*C1File)(nil) _ sanitizeSupportsDiffWriter = (*C1File)(nil) _ sanitizeSyncRunMetadataReader = (*C1File)(nil) ) @@ -228,6 +223,8 @@ type pebbleStore struct { closeMu sync.Mutex closed bool dirty bool + + sourceCacheTest sourceCacheStoreTestSeams } // Compile-time guard: a Pebble store satisfies the full C1ZStore @@ -238,42 +235,12 @@ type pebbleStore struct { // route SQLite *C1File handles today. var _ c1zstore.Store = (*pebbleStore)(nil) -// FileOps overrides the Adapter-level FileOps for two reasons: -// -// - CloneSync threads the pebbleStore's configured payload encoding -// into the destination c1z (otherwise clone output would always -// use the default TAR_ZSTD); and -// - GenerateSyncDiff writes a NEW sync into THIS store, so it must -// flip the dirty bit — without it, Close would skip the envelope -// save and the diff sync would exist only in the discarded temp -// directory. +// FileOps overrides the Adapter-level FileOps so CloneSync threads the +// pebbleStore's configured payload encoding into the destination c1z +// (otherwise clone output would always use the default TAR_ZSTD). +// Clone/isolate write a separate file, so no dirty-marking is needed. func (s *pebbleStore) FileOps() c1zstore.FileOps { - return pebbleStoreFileOps{inner: s.FileOpsWithEncoding(s.payloadEncoding), store: s} -} - -// pebbleStoreFileOps wraps the Adapter-level FileOps to route the one -// mutating-in-place method (GenerateSyncDiff) through the store's -// dirty-marking path. CloneSync writes a separate file and passes -// through unchanged. -type pebbleStoreFileOps struct { - inner c1zstore.FileOps - store *pebbleStore -} - -func (f pebbleStoreFileOps) CloneSync(ctx context.Context, outPath string, syncID string, opts ...c1zstore.CloneSyncOption) error { - return f.inner.CloneSync(ctx, outPath, syncID, opts...) -} - -func (f pebbleStoreFileOps) CopyIsolateSync(ctx context.Context, outPath string, syncID string, opts ...c1zstore.CloneSyncOption) error { - return f.inner.CopyIsolateSync(ctx, outPath, syncID, opts...) -} - -func (f pebbleStoreFileOps) GenerateSyncDiff(ctx context.Context, baseSyncID, appliedSyncID string) (string, error) { - diffSyncID, err := f.inner.GenerateSyncDiff(ctx, baseSyncID, appliedSyncID) - if err != nil { - return "", err - } - return diffSyncID, f.store.markDirty(nil) + return s.FileOpsWithEncoding(s.payloadEncoding) } // SyncMeta overrides the Adapter-level SyncMeta so the MUTATING @@ -364,6 +331,18 @@ func (s *pebbleStore) PebbleEngine() *pebble.Engine { return s.Engine } +func (s *pebbleStore) GrantGenerationDigest(ctx context.Context) (c1zstore.GrantGenerationDigest, bool, error) { + root, ok, err := s.GetGrantDigestGlobalRoot(ctx) + if err != nil || !ok { + return c1zstore.GrantGenerationDigest{}, ok, err + } + return c1zstore.GrantGenerationDigest{ + Hash: append([]byte(nil), root.Hash...), + Count: root.Count, + ABIVersion: pebble.GrantDigestABIVersion, + }, true, nil +} + // CloseEngineOnly closes the Pebble engine without removing the // store's unpacked temp directory, refusing to discard a dirty // writable store. Consumed by the compactor's chunk lifecycle via @@ -498,36 +477,29 @@ func (s *pebbleStore) PutAsset(ctx context.Context, assetRef *v2.AssetRef, conte return s.markDirty(s.Engine.PutAsset(ctx, assetRef, contentType, data)) } -// SetSupportsDiff marks the given sync as diff-capable, matching the -// SQLite engine's sync_runs.supports_diff column. The c1z sanitizer -// carries this marker from a source sync to its sanitized copy so the -// output remains usable wherever the source was. Delegates to the -// SyncMeta sub-store's MarkSyncSupportsDiff. -func (s *pebbleStore) SetSupportsDiff(ctx context.Context, syncID string) error { - return s.markDirty(s.SyncMeta().MarkSyncSupportsDiff(ctx, syncID)) +// PutEntitlementGraphBlob / GetEntitlementGraphBlob / DeleteEntitlementGraphBlob +// expose the entitlement-graph sidecar (see pkg/sync's EntitlementGraphStore). +// The blob format is owned by pkg/sync/expand; the store treats it as opaque. +func (s *pebbleStore) PutEntitlementGraphBlob(ctx context.Context, data []byte) error { + return s.markDirty(s.PutEntitlementGraphSidecar(ctx, data)) } -// SetSyncLink records linkedSyncID as the diff partner of syncID on the -// sync-run record (v3 linked_sync_id), matching the SQLite engine. -// -// This is implemented for connectorstore.Writer parity but is NOT -// reached by the c1z sanitizer's Pebble path: a v3 Pebble c1z holds -// exactly one sync, so there is never a second sync to link to. Cross- -// file linkage is unpreservable on either engine regardless, because -// sanitize mints fresh destination sync ids. -func (s *pebbleStore) SetSyncLink(ctx context.Context, syncID string, linkedSyncID string) error { - if syncID == "" { - return fmt.Errorf("SetSyncLink: empty syncID") - } - r, err := s.GetSyncRunRecord(ctx, syncID) - if err != nil { - return fmt.Errorf("SetSyncLink: get: %w", err) - } - r.SetLinkedSyncId(linkedSyncID) - if err := s.PutSyncRunRecord(ctx, r); err != nil { - return fmt.Errorf("SetSyncLink: put: %w", err) - } - return s.markDirty(nil) +func (s *pebbleStore) GetEntitlementGraphBlob(ctx context.Context) ([]byte, error) { + return s.GetEntitlementGraphSidecar(ctx) +} + +func (s *pebbleStore) DeleteEntitlementGraphBlob(ctx context.Context) error { + return s.markDirty(s.DeleteEntitlementGraphSidecar(ctx)) +} + +// SetSupportsDiff marks the given sync's grant expansion as complete, +// matching the SQLite engine's sync_runs.supports_diff column (the name +// is historical; the marker now gates `baton rollback-expansion`). The +// c1z sanitizer carries this marker from a source sync to its sanitized +// copy so the output remains usable wherever the source was. Delegates +// to the SyncMeta sub-store's MarkSyncSupportsDiff. +func (s *pebbleStore) SetSupportsDiff(ctx context.Context, syncID string) error { + return s.markDirty(s.SyncMeta().MarkSyncSupportsDiff(ctx, syncID)) } func (s *pebbleStore) PutGrants(ctx context.Context, grants ...*v2.Grant) error { @@ -761,6 +733,9 @@ func (g pebbleStoreGrants) ListWithAnnotations(ctx context.Context) iter.Seq2[c1 } func (s *pebbleStore) Close(ctx context.Context) (retErr error) { + if s.sourceCacheTest.beforeCloseLock != nil { + s.sourceCacheTest.beforeCloseLock() + } s.closeMu.Lock() defer s.closeMu.Unlock() if s.closed { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go new file mode 100644 index 00000000..fe1eb55d --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/source_cache.go @@ -0,0 +1,412 @@ +package dotc1z + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + + cdbpebble "github.com/cockroachdb/pebble/v2" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/bid" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// SourceCacheReplayResult reports what one scope's replay copied. +type SourceCacheReplayResult = pebble.SourceCacheReplayResult + +type sourceCacheStoreTestSeams struct { + // afterEngineReplay injects a wrapper-level error after the engine has + // committed replay work. + afterEngineReplay func() error + + // beforeEngineMutation runs after the public wrapper owns closeMu and + // marks dirty, immediately before entering an engine mutation. + beforeEngineMutation func() + + // beforeCloseLock runs immediately before Close attempts to acquire + // closeMu, allowing tests to prove it contends with an active wrapper. + beforeCloseLock func() +} + +// SourceCacheStore is the optional store capability backing source-cache +// replay (see proto/c1/connector/v2/annotation_source_cache.proto). It is +// implemented ONLY by the Pebble engine; the syncer type-asserts for it and +// treats a store without it as "source cache unsupported" (no-op lookup, +// no replay). It is deliberately NOT part of c1zstore.Store. +type SourceCacheStore interface { + // LookupSourceCacheEntry returns this store's manifest entry for + // (kind, scopeKey). Backs the connector-facing lookup when this + // store is the previous sync. + LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string) (sourcecache.Entry, bool, error) + + // PutSourceCacheEntry writes the current sync's manifest entry for + // (kind, scopeKey). Zero-row scopes still get entries. + PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string, cacheValidator string) error + + // ReplaySourceCache copies every row stamped with scopeKey from prev + // (the previous sync's store, opened read-only) into this store. prev + // must be a Pebble store. Does NOT write the manifest entry — the + // caller writes it after the scope's overlay/deletes complete, so a + // failed replay can't leave a phantom hit for the next sync. + // + // On error, the result's Rows reports rows whose bounded intermediate + // commits landed (retry converges; matching the delete siblings + // below), and NeedsExpansion may overreport a staged-but-uncommitted + // row — the safe direction, since arming expansion is idempotent and + // add-only. + ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeKey string) (SourceCacheReplayResult, error) + + // DeleteSourceCacheRows removes rows by public canonical ID from the + // current sync, after replay + overlay (delta-query tombstones). + // ID formats per kind: grants and entitlements use their canonical + // IDs; resources use Baton resource BIDs ("bid:r:..."). + // + // scopeKey is the scope whose delta reported the tombstones. The + // deletes act FOR that scope: removing its own rows is the + // legitimate shrink flow and stages no poison, while removing a row + // stamped with any OTHER scope is a row-partition violation that + // durably poisons the stamped scope (CO-015 — it becomes a lookup + // miss and is refused as a replay source). + // + // Grant resolution is BOUNDED: candidate probing only, never the + // O(all grants) stored-external-id scan. Grants stored under + // connector-custom ids are unreachable here (delete no-ops); such + // connectors use DeleteSourceCacheRowsInScope instead. Deletes commit + // in bounded chunks; deletion is idempotent, so retry converges. + DeleteSourceCacheRows(ctx context.Context, kind sourcecache.RowKind, scopeKey string, ids []string) error + + // DeleteSourceCacheRowsInScope removes rows stamped with scopeKey by + // bare object id — grants by principal id (no principal type, no + // canonical-id reconstruction), resources by resource id (any type). + // One index scan of the scope per call; a page's tombstones are + // batched into one call. Deletes commit in bounded chunks; on error, + // the returned count reports rows already committed and retry converges. + // Ids with no matching rows are no-ops. + // Not supported for entitlements. + DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeKey string, ids []string) (int64, error) + + // DeleteSourceCacheGrantsByIDInScope removes grant rows stamped with + // scopeKey whose STORED grant id is in ids — works for + // connector-custom grant-id shapes that the global bounded delete + // cannot resolve, and stays bounded by the scope's row count. Ids with + // no matching rows are no-ops. Deletes commit in bounded chunks; on + // error, the returned count reports rows already committed. + DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeKey string, ids []string) (int64, error) +} + +var _ SourceCacheStore = (*pebbleStore)(nil) + +func (s *pebbleStore) beginSourceCacheMutation() (func(), error) { + s.closeMu.Lock() + if s.closed { + s.closeMu.Unlock() + return nil, pebble.ErrEngineClosing + } + // Hold closeMu until the engine call and dirty transition are complete. + // Close cannot checkpoint between these two halves of one public mutation. + s.dirty = true + if s.sourceCacheTest.beforeEngineMutation != nil { + s.sourceCacheTest.beforeEngineMutation() + } + return s.closeMu.Unlock, nil +} + +// sourceCacheEngine recovers the Pebble engine from an arbitrary store, +// nil-safe. Mirrors pebble.AsEngine but accepts any value so the syncer +// can probe its previous-sync reader without caring about its static type. +func sourceCacheEngine(store any) (*pebble.Engine, bool) { + a, ok := store.(interface{ PebbleEngine() *pebble.Engine }) + if !ok { + return nil, false + } + e := a.PebbleEngine() + return e, e != nil +} + +func validateReplaySourceEligible(ctx context.Context, previous *pebble.Engine) error { + run, err := previous.LatestFinishedSyncRecord(ctx, nil) + if err != nil { + return fmt.Errorf("source cache replay: read previous sync lifecycle: %w", err) + } + if run == nil { + return errors.New("source cache replay: previous artifact sync is not finished") + } + if run.GetType() != v3.SyncType_SYNC_TYPE_FULL { + return fmt.Errorf("source cache replay: previous artifact sync type %s is not replay-eligible", run.GetType()) + } + if run.GetCompacted() { + return errors.New("source cache replay: compacted artifacts are not replay-eligible") + } + return nil +} + +func sameSourceCacheArtifact(current *pebbleStore, previous connectorstore.Reader) bool { + prev, ok := previous.(*pebbleStore) + if !ok { + return false + } + currentPath, currentErr := filepath.Abs(current.outputFilePath) + previousPath, previousErr := filepath.Abs(prev.outputFilePath) + if currentErr == nil && previousErr == nil && filepath.Clean(currentPath) == filepath.Clean(previousPath) { + return true + } + currentInfo, currentErr := os.Stat(current.outputFilePath) + previousInfo, previousErr := os.Stat(prev.outputFilePath) + return currentErr == nil && previousErr == nil && os.SameFile(currentInfo, previousInfo) +} + +func (s *pebbleStore) LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string) (sourcecache.Entry, bool, error) { + if err := sourcecache.ValidateRowKind(kind); err != nil { + return sourcecache.Entry{}, false, err + } + if err := sourcecache.ValidateScopeKey(scopeKey); err != nil { + return sourcecache.Entry{}, false, err + } + rec, err := s.GetSourceCacheEntry(ctx, string(kind), scopeKey) + if err != nil { + if errors.Is(err, cdbpebble.ErrNotFound) { + return sourcecache.Entry{}, false, nil + } + return sourcecache.Entry{}, false, err + } + if rec.GetInvalidated() || rec.GetCacheValidator() == "" { + return sourcecache.Entry{}, false, nil + } + // A poisoned scope reads as a MISS (CO-015): this store observed a + // row-partition violation against it, so its stamped row set no + // longer matches what the validator vouches for. Reporting a miss + // makes the scope re-fetch cold and converge; reporting a hit would + // send orchestration into a replay that preflight hard-refuses. + poisoned, err := s.SourceCachePoisoned(ctx, string(kind), scopeKey) + if err != nil { + return sourcecache.Entry{}, false, err + } + if poisoned { + return sourcecache.Entry{}, false, nil + } + return sourcecache.Entry{ + CacheValidator: rec.GetCacheValidator(), + DiscoveredAt: rec.GetDiscoveredAt().AsTime(), + }, true, nil +} + +func (s *pebbleStore) PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string, cacheValidator string) error { + if err := sourcecache.ValidateRowKind(kind); err != nil { + return err + } + if err := sourcecache.ValidateScopeKey(scopeKey); err != nil { + return err + } + if cacheValidator == "" { + return errors.New("source cache manifest: cache validator is required") + } + done, err := s.beginSourceCacheMutation() + if err != nil { + return err + } + defer done() + return s.Engine.PutSourceCacheEntry(ctx, string(kind), scopeKey, cacheValidator) +} + +func (s *pebbleStore) ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeKey string) (SourceCacheReplayResult, error) { + if err := sourcecache.ValidateRowKind(kind); err != nil { + return SourceCacheReplayResult{}, err + } + if err := sourcecache.ValidateScopeKey(scopeKey); err != nil { + return SourceCacheReplayResult{}, err + } + if sameSourceCacheArtifact(s, prev) { + return SourceCacheReplayResult{}, errors.New("source cache replay: previous and current stores use the same artifact") + } + prevEngine, ok := sourceCacheEngine(prev) + if !ok { + return SourceCacheReplayResult{}, errors.New("source cache replay: previous sync store is not a pebble store") + } + if prevEngine == s.Engine { + return SourceCacheReplayResult{}, errors.New("source cache replay: previous and current stores are the same") + } + if err := validateReplaySourceEligible(ctx, prevEngine); err != nil { + return SourceCacheReplayResult{}, err + } + entry, err := prevEngine.GetSourceCacheEntry(ctx, string(kind), scopeKey) + if err != nil { + if errors.Is(err, cdbpebble.ErrNotFound) { + return SourceCacheReplayResult{}, fmt.Errorf( + "source cache replay: no manifest for row kind %q and scope %q: %w", + kind, + scopeKey, + cdbpebble.ErrNotFound, + ) + } + return SourceCacheReplayResult{}, fmt.Errorf("source cache replay: read previous manifest: %w", err) + } + if entry.GetInvalidated() { + return SourceCacheReplayResult{}, fmt.Errorf("source cache replay: manifest for row kind %q and scope %q is invalidated", kind, scopeKey) + } + if entry.GetCacheValidator() == "" { + return SourceCacheReplayResult{}, fmt.Errorf("source cache replay: manifest for row kind %q and scope %q has no validator", kind, scopeKey) + } + // Replay is replacement, not append: the engine may clear destination rows + // or commit one or more bounded chunks before returning zero rows or an error. + // Serialize dirty marking and the engine mutation against Close so a checkpoint + // cannot cut between them. + done, err := s.beginSourceCacheMutation() + if err != nil { + return SourceCacheReplayResult{}, err + } + defer done() + var res SourceCacheReplayResult + switch kind { + case sourcecache.RowKindResources: + res, err = s.ReplaySourceCacheResources(ctx, prevEngine, scopeKey) + case sourcecache.RowKindEntitlements: + res, err = s.ReplaySourceCacheEntitlements(ctx, prevEngine, scopeKey) + case sourcecache.RowKindGrants: + res, err = s.ReplaySourceCacheGrants(ctx, prevEngine, scopeKey) + default: + return SourceCacheReplayResult{}, fmt.Errorf("source cache replay: invalid row kind %q", kind) + } + if err != nil { + // Committed progress rides the error: bounded intermediate chunks + // may have landed before the failure, and the engine result + // reports exactly those rows — same contract as the delete + // siblings on this interface. Zeroing it here would make the + // engine-level contract unreachable from the only surface replay + // orchestration uses. + return res, err + } + if s.sourceCacheTest.afterEngineReplay != nil { + if err := s.sourceCacheTest.afterEngineReplay(); err != nil { + return res, err + } + } + return res, nil +} + +// DeleteSourceCacheRows deletes delta tombstones by public id string. +// +// NOTE on the bare-id lookup safety contract (engine/pebble/lookup.go): +// sync paths normally must not resolve grants by string. This path is a +// deliberate, narrow exception: tombstone ids are strings the connector +// itself emitted for these rows, volumes are delta-sized (not O(rows)), +// and resolution keeps the exactly-one rule — an ambiguous id fails the +// sync loudly rather than guessing a delete, which matches the +// source-cache replay-phase error policy. +func (s *pebbleStore) DeleteSourceCacheRows(ctx context.Context, kind sourcecache.RowKind, scopeKey string, ids []string) error { + if err := sourcecache.ValidateRowKind(kind); err != nil { + return err + } + if err := sourcecache.ValidateScopeKey(scopeKey); err != nil { + return err + } + if len(ids) == 0 { + return nil + } + if kind == sourcecache.RowKindResources { + refs := make([]pebble.ResourceRef, len(ids)) + for i, id := range ids { + r, err := bid.ParseResourceBid(id) + if err != nil { + return fmt.Errorf("source cache delete resource: invalid resource bid %q: %w", id, err) + } + refs[i] = pebble.ResourceRef{ + ResourceTypeID: r.GetId().GetResourceType(), + ResourceID: r.GetId().GetResource(), + } + } + done, err := s.beginSourceCacheMutation() + if err != nil { + return err + } + defer done() + if err := s.DeleteResourceRecordsBounded(ctx, refs, scopeKey); err != nil { + return fmt.Errorf("source cache delete resources for scope %q: %w", scopeKey, err) + } + return nil + } + done, err := s.beginSourceCacheMutation() + if err != nil { + return err + } + defer done() + switch kind { + case sourcecache.RowKindGrants: + if err := s.DeleteGrantRecordsBounded(ctx, ids, scopeKey); err != nil { + return fmt.Errorf("source cache delete grants: %w", err) + } + case sourcecache.RowKindEntitlements: + if err := s.DeleteEntitlementRecords(ctx, ids, scopeKey); err != nil { + return fmt.Errorf("source cache delete entitlements: %w", err) + } + case sourcecache.RowKindResources: + return errors.New("source cache delete resource: internal dispatch error") + } + return nil +} + +func (s *pebbleStore) DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeKey string, ids []string) (int64, error) { + if err := sourcecache.ValidateScopeKey(scopeKey); err != nil { + return 0, err + } + if len(ids) == 0 { + return 0, nil + } + idSet := make(map[string]struct{}, len(ids)) + for _, id := range ids { + idSet[id] = struct{}{} + } + done, err := s.beginSourceCacheMutation() + if err != nil { + return 0, err + } + defer done() + deleted, err := s.DeleteGrantsByExternalIDsInScope(ctx, scopeKey, idSet) + if err != nil { + return deleted, fmt.Errorf("source cache grant-id delete for scope %q: %w", scopeKey, err) + } + return deleted, nil +} + +func (s *pebbleStore) DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeKey string, ids []string) (int64, error) { + if err := sourcecache.ValidateRowKind(kind); err != nil { + return 0, err + } + if err := sourcecache.ValidateScopeKey(scopeKey); err != nil { + return 0, err + } + if len(ids) == 0 { + return 0, nil + } + idSet := make(map[string]struct{}, len(ids)) + for _, id := range ids { + idSet[id] = struct{}{} + } + if kind == sourcecache.RowKindEntitlements { + return 0, fmt.Errorf("source cache scoped delete: not supported for entitlements") + } + done, err := s.beginSourceCacheMutation() + if err != nil { + return 0, err + } + defer done() + var deleted int64 + // A matching orphan scope index is a durable mutation even though no + // primary row contributes to the returned deletion count. + switch kind { + case sourcecache.RowKindGrants: + deleted, err = s.DeleteGrantsByPrincipalsInScope(ctx, scopeKey, idSet) + case sourcecache.RowKindResources: + deleted, err = s.DeleteResourcesByIDsInScope(ctx, scopeKey, idSet) + case sourcecache.RowKindEntitlements: + return 0, fmt.Errorf("source cache scoped delete: not supported for entitlements") + } + if err != nil { + return deleted, fmt.Errorf("source cache scoped delete for scope %q: %w", scopeKey, err) + } + return deleted, nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go index 18a7fe5e..fe34588a 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sql_helpers.go @@ -352,7 +352,8 @@ func listConnectorObjects[T proto.Message](ctx context.Context, c *C1File, table return ret, nextPageToken, nil } -// This is required for sync diffs to work. Its not much slower. +// Deterministic marshaling keeps stored blobs byte-comparable across +// writes (compaction change detection, digests). Its not much slower. var protoMarshaler = proto.MarshalOptions{Deterministic: true} // prepareSingleConnectorObjectRow processes a single message and returns the prepared record. @@ -424,7 +425,8 @@ func prepareConnectorObjectRowsParallel[T proto.Message]( protoMarshallers := make([]proto.MarshalOptions, numWorkers) for i := range numWorkers { - // Deterministic marshaling is required for sync diffs to work. Its not much slower. + // Deterministic marshaling keeps stored blobs byte-comparable + // across writes. Its not much slower. protoMarshallers[i] = proto.MarshalOptions{Deterministic: true} } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go index 61af474b..8bc0bf27 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/sync_runs.go @@ -107,7 +107,12 @@ func (r *syncRunsTable) Migrations(ctx context.Context, db *goqu.Database) (bool migrated = true } - // Check if linked_sync_id column exists + // linked_sync_id is vestigial: only the removed diff-sync feature ever + // wrote non-empty values, and nothing reads it anymore. The column stays + // in the schema (and this migration stays) so every opened file has a + // uniform sync_runs shape — CloneSync/SnapshotTo copy rows using the + // source's PRAGMA table_info column list into a fresh-DDL destination, + // and older SDKs still SELECT the column by name. var linkedSyncIDExists int err = db.QueryRowContext(ctx, fmt.Sprintf("select count(*) from pragma_table_info('%s') where name='linked_sync_id'", r.Name())).Scan(&linkedSyncIDExists) if err != nil { @@ -266,7 +271,7 @@ func (c *C1File) getLatestUnfinishedSync(ctx context.Context, syncType connector q := c.db.From(syncRuns.Name()) q = q.Select( "sync_id", "started_at", "ended_at", "sync_token", "sync_type", - "parent_sync_id", "linked_sync_id", "supports_diff", + "parent_sync_id", "supports_diff", "ingest_invariant_generation", "ingest_invariant_coverage", "ingest_invariant_mode", "stats", ) @@ -288,7 +293,7 @@ func (c *C1File) getLatestUnfinishedSync(ctx context.Context, syncType connector var generation, coverageJSON, mode string err = row.Scan( &ret.ID, &ret.StartedAt, &ret.EndedAt, &ret.SyncToken, &ret.Type, - &ret.ParentSyncID, &ret.LinkedSyncID, &ret.SupportsDiff, + &ret.ParentSyncID, &ret.SupportsDiff, &generation, &coverageJSON, &mode, &statsBytes, ) if err != nil { @@ -323,7 +328,7 @@ func (c *C1File) getFinishedSync(ctx context.Context, offset uint, syncType conn q := c.db.From(syncRuns.Name()) q = q.Select( "sync_id", "started_at", "ended_at", "sync_token", "sync_type", - "parent_sync_id", "linked_sync_id", "supports_diff", + "parent_sync_id", "supports_diff", "ingest_invariant_generation", "ingest_invariant_coverage", "ingest_invariant_mode", "stats", ) @@ -352,7 +357,7 @@ func (c *C1File) getFinishedSync(ctx context.Context, offset uint, syncType conn var generation, coverageJSON, mode string err = row.Scan( &ret.ID, &ret.StartedAt, &ret.EndedAt, &ret.SyncToken, &ret.Type, - &ret.ParentSyncID, &ret.LinkedSyncID, &ret.SupportsDiff, + &ret.ParentSyncID, &ret.SupportsDiff, &generation, &coverageJSON, &mode, &statsBytes, ) if err != nil { @@ -398,7 +403,7 @@ func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize ui q := c.db.From(syncRuns.Name()).Prepared(true) q = q.Select( "id", "sync_id", "started_at", "ended_at", "sync_token", "sync_type", - "parent_sync_id", "linked_sync_id", "supports_diff", + "parent_sync_id", "supports_diff", "ingest_invariant_generation", "ingest_invariant_coverage", "ingest_invariant_mode", "stats", ) @@ -440,7 +445,7 @@ func (c *C1File) ListSyncRuns(ctx context.Context, pageToken string, pageSize ui var generation, coverageJSON, mode string err := rows.Scan( &rowId, &data.ID, &data.StartedAt, &data.EndedAt, &data.SyncToken, &data.Type, - &data.ParentSyncID, &data.LinkedSyncID, &data.SupportsDiff, + &data.ParentSyncID, &data.SupportsDiff, &generation, &coverageJSON, &mode, &statsBytes, ) if err != nil { @@ -542,7 +547,7 @@ func (c *C1File) getSync(ctx context.Context, syncID string) (*c1zstore.SyncRun, q := c.db.From(syncRuns.Name()) q = q.Select( "sync_id", "started_at", "ended_at", "sync_token", "sync_type", - "parent_sync_id", "linked_sync_id", "supports_diff", + "parent_sync_id", "supports_diff", "ingest_invariant_generation", "ingest_invariant_coverage", "ingest_invariant_mode", "stats", ) @@ -557,7 +562,7 @@ func (c *C1File) getSync(ctx context.Context, syncID string) (*c1zstore.SyncRun, var generation, coverageJSON, mode string err = row.Scan( &ret.ID, &ret.StartedAt, &ret.EndedAt, &ret.SyncToken, &ret.Type, - &ret.ParentSyncID, &ret.LinkedSyncID, &ret.SupportsDiff, + &ret.ParentSyncID, &ret.SupportsDiff, &generation, &coverageJSON, &mode, &statsBytes, ) if err != nil { @@ -757,10 +762,6 @@ func (c *C1File) StartNewSync(ctx context.Context, syncType connectorstore.SyncT return "", status.Errorf(codes.InvalidArgument, "parent sync id must be empty for resources only sync") } case connectorstore.SyncTypePartial: - case connectorstore.SyncTypePartialUpserts, connectorstore.SyncTypePartialDeletions: - // Diff syncs carry the base sync as their parent; the linked - // pairing (upserts ↔ deletions) is set separately via - // SetSyncLink since the partner's id may not exist yet. case connectorstore.SyncTypeAny: return "", status.Errorf(codes.InvalidArgument, "sync cannot be started with SyncTypeAny") default: @@ -780,10 +781,6 @@ func (c *C1File) StartNewSync(ctx context.Context, syncType connectorstore.SyncT } func (c *C1File) insertSyncRun(ctx context.Context, syncID string, syncType connectorstore.SyncType, parentSyncID string) error { - return c.insertSyncRunWithLink(ctx, syncID, syncType, parentSyncID, "") -} - -func (c *C1File) insertSyncRunWithLink(ctx context.Context, syncID string, syncType connectorstore.SyncType, parentSyncID string, linkedSyncID string) error { if c.readOnly { return ErrReadOnly } @@ -800,7 +797,6 @@ func (c *C1File) insertSyncRunWithLink(ctx context.Context, syncID string, syncT "sync_token": "", "sync_type": syncType, "parent_sync_id": parentSyncID, - "linked_sync_id": linkedSyncID, "grants_backfilled": 1, // New syncs do not require grants backfill. }) @@ -883,8 +879,10 @@ func (c *C1File) endSyncRun(ctx context.Context, syncID string) error { return nil } -// SetSupportsDiff marks the given sync as supporting diff operations. -// This indicates the sync has SQL-layer grant metadata (is_expandable) properly populated. +// SetSupportsDiff marks the given sync's data collection as complete with +// SQL-layer grant metadata (is_expandable) properly populated. The name is +// historical (the marker once gated diff-sync generation); today it gates +// `baton rollback-expansion`, which refuses syncs without the marker. func (c *C1File) SetSupportsDiff(ctx context.Context, syncID string) error { ctx, span := tracer.Start(ctx, "C1File.SetSupportsDiff") var err error @@ -1000,43 +998,6 @@ func (c *C1File) clearIngestInvariantVerification(ctx context.Context, syncID st return nil } -// SetSyncLink sets the linked_sync_id of an existing sync run. Diff -// sync pairs (partial_upserts ↔ partial_deletions) reference each -// other bidirectionally; a writer rebuilding such a pair cannot supply -// the link at StartNewSync time because the partner's id is minted by -// the store, so the pairing is applied after both runs exist. -func (c *C1File) SetSyncLink(ctx context.Context, syncID string, linkedSyncID string) error { - ctx, span := tracer.Start(ctx, "C1File.SetSyncLink") - var err error - defer func() { uotel.EndSpanWithError(span, err) }() - - if c.readOnly { - return ErrReadOnly - } - if syncID == "" { - return status.Errorf(codes.InvalidArgument, "sync id is required") - } - - q := c.db.Update(syncRuns.Name()) - q = q.Set(goqu.Record{ - "linked_sync_id": linkedSyncID, - }) - q = q.Where(goqu.C("sync_id").Eq(syncID)) - - query, args, err := q.ToSQL() - if err != nil { - return err - } - - _, err = c.db.ExecContext(ctx, query, args...) - if err != nil { - return err - } - c.dbUpdated.Store(true) - - return nil -} - // When context deadline is exceeded, go-sqlite can return a SQLITE_INTERRUPT error. // If that happens, wrapSqliteInterruptError wraps this error and returns context.DeadlineExceeded. // This allows sync cleanup to return ErrSyncNotComplete and resume its work on the next run. diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go index 91d5c1fe..ede01029 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/dotc1z/to_pebble.go @@ -200,9 +200,8 @@ type syncIDPreservingStarter interface { // // When the source has no sync runs at all, "" writes an empty Pebble c1z (so // convert-open succeeds on never-synced files). If sync runs exist but none -// match the selected resolve behavior (e.g. diff-only under Newest), "" -// returns an error rather than discarding data. Pass an explicit syncID to -// convert a specific sync (including diff syncs for fixture seeding). The +// match the selected resolve behavior, "" returns an error rather than +// discarding data. Pass an explicit syncID to convert a specific sync. The // destination sync is written ended when the source was finished; when the // source was unfinished, EndSync still runs (indexes/digests/stats/flush) but // ended_at is cleared and the source sync_token is preserved so the sync @@ -214,11 +213,10 @@ type syncIDPreservingStarter interface { // since nothing resumes a sealed sync and the connector lifecycle deletes them // at that point anyway. // -// The sync's lineage columns — parent_sync_id, linked_sync_id, supports_diff — -// are preserved too. They reference syncs that the single-sync destination -// cannot hold, but those references are meaningful across files: dropping them -// would make a converted partial read as a standalone snapshot and a -// diff-capable sync read as non-diffable. +// The sync's lineage columns — parent_sync_id, supports_diff — are preserved +// too. The parent reference names a sync the single-sync destination cannot +// hold, but it is meaningful across files: dropping it would make a converted +// partial read as a standalone snapshot. // // The Pebble engine is registered statically with dotc1z; no extra // imports are needed before calling. @@ -399,7 +397,6 @@ func (c *C1File) ToPebble(ctx context.Context, outPath string, syncID string, op if err != nil { return nil, fmt.Errorf("to-pebble: load destination sync metadata: %w", err) } - rec.SetLinkedSyncId(sync.LinkedSyncID) rec.SetSupportsDiff(sync.SupportsDiff) // Localized on the way in: these scanned wall clocks become absolute // instants in the Pebble record, and Pebble's resume cutoff compares @@ -459,9 +456,7 @@ func (c *C1File) ToPebble(ctx context.Context, outPath string, syncID string, op } // discardedSyncs lists the source syncs a conversion that keeps keepSyncID -// leaves behind, in sync_runs order. Diff-pair syncs are included: they are -// dropped from the artifact too, and their absence is what an operator chasing -// a missing delta needs to see. +// leaves behind, in sync_runs order. // // Metadata only. ListSyncRuns reads the sync_runs rows and parses the cached // stats blob when one is present; unlike GetSync it never recomputes stats, so @@ -532,14 +527,6 @@ func discardedSyncFields(discarded []DiscardedSync) []zap.Field { // chosen when it is all there is (newest started_at among them), since // convert-open must not fail on such a file. Unfinished syncs within the // cutoff are live work and keep competing on started_at alone. -// -// The excluded types are the diff pair written by attached-file diffing, -// partial_upserts and partial_deletions: each holds one side of a delta and -// is meaningless converted alone. GenerateSyncDiff's delta sync is NOT -// excluded — it is stored as a plain partial (diff.go), indistinguishable -// from a targeted sync by type, parent_sync_id, or supports_diff — so on a -// file that was just diffed and holds no newer sync, "" resolves to the -// delta. func (c *C1File) resolveConvertSyncID(ctx context.Context) (string, error) { q := c.db.From(syncRuns.Name()).Prepared(true). Select("sync_id"). diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/exit/exit.go b/vendor/github.com/conductorone/baton-sdk/pkg/exit/exit.go new file mode 100644 index 00000000..225de5a9 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/exit/exit.go @@ -0,0 +1,53 @@ +package exit + +import ( + "context" + "errors" + "fmt" + "os" + + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// Exit exits the program with a code based on the error. +// Exit codes correspond to GRPC status codes. +// If err is nil, exit code is 0. +// Common errors such as context cancelled and deadline exceeded exit with the corresponding GRPC status code. +// Other errors exit with code 2, which is GRPC status code Unknown. +func Exit(err error) { + os.Exit(exitCode(err)) +} + +// LogExit logs the error to stderr & calls Exit(), which exits the program with a code based on the error. +func LogExit(err error) { + if err != nil { + fmt.Fprintln(os.Stderr, err.Error()) + } + Exit(err) +} + +func exitCode(err error) int { + if err == nil { + return 0 + } + + if grpcErr, ok := status.FromError(err); ok { + if grpcErr.Code() == codes.OK { + // An error with code OK should never happen. + return int(codes.Internal) + } + return int(grpcErr.Code()) + } + + if errors.Is(err, context.Canceled) { + return int(codes.Canceled) + } + + if errors.Is(err, context.DeadlineExceeded) { + return int(codes.DeadlineExceeded) + } + + // Otherwise, exit with code 2, which is GRPC status code Unknown. + return int(codes.Unknown) +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/field/default_relationships.go b/vendor/github.com/conductorone/baton-sdk/pkg/field/default_relationships.go index f859d8fc..5da905ef 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/field/default_relationships.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/field/default_relationships.go @@ -6,7 +6,6 @@ var DefaultRelationships = []SchemaFieldRelationship{ FieldsRequiredTogether(createTicketField, ticketTemplatePathField), FieldsRequiredTogether(bulkCreateTicketField, bulkTicketTemplatePathField), FieldsRequiredTogether(getTicketField, ticketIDField), - FieldsRequiredTogether(diffSyncsField, diffSyncsBaseSyncField, diffSyncsAppliedSyncField), FieldsRequiredTogether(compactSyncsField, compactSyncIDsField, compactFilePathsField, compactOutputDirectoryField), FieldsMutuallyExclusive( grantEntitlementField, @@ -27,7 +26,6 @@ var DefaultRelationships = []SchemaFieldRelationship{ deleteResourceTypeField, rotateCredentialsTypeField, eventFeedField, - diffSyncsField, compactSyncsField, ListTicketSchemasField, ), diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go b/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go index 16857042..29e60583 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/field/defaults.go @@ -136,25 +136,6 @@ var ( WithDescription("The resource type IDs to sync"), WithPersistent(true), WithExportTarget(ExportTargetNone)) - diffSyncsField = BoolField( - "diff-syncs", - WithDescription("Create a new partial SyncID from a base and applied sync."), - WithHidden(true), - WithPersistent(true), - WithExportTarget(ExportTargetNone), - ) - diffSyncsBaseSyncField = StringField("base-sync-id", - WithDescription("The base sync to diff from."), - WithHidden(true), - WithPersistent(true), - WithExportTarget(ExportTargetNone), - ) - diffSyncsAppliedSyncField = StringField("applied-sync-id", - WithDescription("The sync to show diffs when applied to the base sync."), - WithHidden(true), - WithPersistent(true), - WithExportTarget(ExportTargetNone), - ) compactSyncsField = BoolField("compact-syncs", WithDescription("Provide a list of sync files to compact into a single c1z file and sync ID."), @@ -355,10 +336,10 @@ var ( })) // StorageEngineField selects the dotc1z storage engine for sync tasks. - // Empty uses the baton-sdk default (sqlite for new files). + // Empty uses the baton-sdk default (pebble for new files). StorageEngineField = StringField("storage-engine", WithDescription("The storage engine to use when opening the sync c1z file: sqlite or pebble. "+ - "Leave unset to use the baton-sdk default."), + "Defaults to pebble when unset."), WithPersistent(true), WithExportTarget(ExportTargetNone), WithString(func(r *StringRuler) { @@ -439,9 +420,6 @@ var DefaultFields = append([]SchemaField{ externalResourceEntitlementIdFilter, externalResourceTraitsField, KeepPreviousSyncC1ZField, - diffSyncsField, - diffSyncsBaseSyncField, - diffSyncsAppliedSyncField, compactSyncIDsField, compactFilePathsField, compactOutputDirectoryField, diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/provisioner/provisioner.go b/vendor/github.com/conductorone/baton-sdk/pkg/provisioner/provisioner.go index ca63af0d..5693e268 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/provisioner/provisioner.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/provisioner/provisioner.go @@ -9,6 +9,7 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.opentelemetry.io/otel" "go.uber.org/zap" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/structpb" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" @@ -126,6 +127,31 @@ func (p *Provisioner) Close(ctx context.Context) error { return nil } +// hydrateEntitlementResource returns a copy of e with Resource replaced by +// resource. The store's entitlement read (GetEntitlement, via +// V3EntitlementToV2 on the Pebble engine) returns Resource as an +// identity-only stub; callers that already fetched the full Resource +// separately (e.g. for the external-resource annotation check) must +// splice it back in before handing the entitlement to a connector's +// Grant/Revoke, or the connector never sees the resource's Profile, +// DisplayName, etc. +// +// GrantableTo is untouched: V3EntitlementToV2 also stubs it down to +// ResourceType id-only entries, and this helper does not re-hydrate those — +// a connector reading entitlement.GrantableTo display names/traits in +// Grant/Revoke still sees stubs on the Pebble engine. +func hydrateEntitlementResource(e *v2.Entitlement, resource *v2.Resource) *v2.Entitlement { + if e == nil { + return nil + } + hydrated, ok := proto.Clone(e).(*v2.Entitlement) + if !ok { + return e + } + hydrated.SetResource(resource) + return hydrated +} + func (p *Provisioner) grant(ctx context.Context) error { ctx, span := tracer.Start(ctx, "Provisioner.grant") var err error @@ -165,18 +191,9 @@ func (p *Provisioner) grant(ctx context.Context) error { return err } - resource := v2.Resource_builder{ - Id: principal.GetResource().GetId(), - DisplayName: principal.GetResource().GetDisplayName(), - Annotations: principal.GetResource().GetAnnotations(), - Description: principal.GetResource().GetDescription(), - ExternalId: principal.GetResource().GetExternalId(), //nolint:staticcheck // Deprecated. - ParentResourceId: principal.GetResource().GetParentResourceId(), - }.Build() - _, err = p.connector.Grant(ctx, v2.GrantManagerServiceGrantRequest_builder{ - Entitlement: entitlement.GetEntitlement(), - Principal: resource, + Entitlement: hydrateEntitlementResource(entitlement.GetEntitlement(), entitlementResource.GetResource()), + Principal: principal.GetResource(), }.Build()) if err != nil { return err @@ -228,20 +245,11 @@ func (p *Provisioner) revoke(ctx context.Context) error { return errors.New("cannot revoke grant on external resource") } - resource := v2.Resource_builder{ - Id: principal.GetResource().GetId(), - DisplayName: principal.GetResource().GetDisplayName(), - Annotations: principal.GetResource().GetAnnotations(), - Description: principal.GetResource().GetDescription(), - ExternalId: principal.GetResource().GetExternalId(), //nolint:staticcheck // Deprecated. - ParentResourceId: principal.GetResource().GetParentResourceId(), - }.Build() - _, err = p.connector.Revoke(ctx, v2.GrantManagerServiceRevokeRequest_builder{ Grant: v2.Grant_builder{ Id: grant.GetGrant().GetId(), - Entitlement: entitlement.GetEntitlement(), - Principal: resource, + Entitlement: hydrateEntitlementResource(entitlement.GetEntitlement(), entitlementResource.GetResource()), + Principal: principal.GetResource(), Annotations: grant.GetGrant().GetAnnotations(), }.Build(), }.Build()) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go index a03cb764..00d0d815 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sdk/version.go @@ -1,3 +1,3 @@ package sdk -const Version = "v0.24.3" +const Version = "v0.25.1" diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/context.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/context.go new file mode 100644 index 00000000..c0333ca8 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/context.go @@ -0,0 +1,17 @@ +package sourcecache + +import "context" + +type scopeContextKey struct{} + +// WithScope returns a context carrying the source-cache scope key for rows +// written under it. +func WithScope(ctx context.Context, scopeKey string) context.Context { + return context.WithValue(ctx, scopeContextKey{}, scopeKey) +} + +// ScopeFromContext returns the scope key set by WithScope, or "". +func ScopeFromContext(ctx context.Context) string { + scopeKey, _ := ctx.Value(scopeContextKey{}).(string) + return scopeKey +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go new file mode 100644 index 00000000..d3b3c5fd --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sourcecache/sourcecache.go @@ -0,0 +1,148 @@ +// Package sourcecache defines the connector-facing surface of source-cache +// replay (see proto/c1/connector/v2/annotation_source_cache.proto). +// +// NOT YET WIRED. This package describes the intended contract, and the +// storage and eligibility machinery beneath it is in place, but the syncer +// does not install a Lookup or consume these annotations yet: SyncOpAttrs +// carries no source-cache field, and nothing outside this package references +// Lookup, SetLookup, or NoopLookup. A connector written against the surface +// below will compile and do nothing until the orchestration lands. The +// wiring described here is present tense on purpose — it is the contract the +// orchestration must satisfy, not a description of today's behavior. +// +// A connector that can cheaply revalidate upstream data — HTTP conditional +// requests (GitHub), delta queries (Microsoft Graph) — opts in by attaching +// SourceCacheCapability MODE_READ_WRITE to its Validate response. During a +// sync it looks up the previous validator for a scope via the Lookup the SDK +// provides on SyncOpAttrs, revalidates upstream, and either emits fresh rows +// tagged with SourceCacheRecord or asks the SDK to replay the previous rows +// with SourceCacheReplay. +// +// The connector owns scope computation; the SDK only keys storage by the +// connector-supplied scope key. The validator (etag, delta token) is opaque +// to the SDK. +// +// Invariant that keeps replay safe: a connector must only emit +// SourceCacheReplay for a scope whose validator it received from THIS sync's +// Lookup. The lookup need not happen in the same call that emits the +// replay: a planning call may batch-resolve many scopes and pass the +// verdicts to sibling cursors through EnqueuePageTokens page tokens — that +// satisfies the invariant, because the validator still originates from the +// consuming sync. What's forbidden is a validator that outlives a sync +// (connector-side caches, config, upstream echoes). When source cache is +// disabled or degraded (no capability, no usable previous sync, unsupported +// storage engine) the SDK installs NoopLookup, every lookup misses, and a +// well-behaved connector naturally falls back to full fetch. +// +// Replay equivalence: a cached sync must reproduce what a full resync +// would produce. Replayed rows are verbatim copies of the previous sync's +// rows with one deliberate exception — expander-written Sources on direct +// grants (classified by a self-source entry, mirroring RollbackExpansion) +// are stripped at copy time so the current sync's expansion recomputes +// them from true state; re-expansion only adds contributions, so carrying +// them verbatim would immortalize contributions removed upstream. +// Connector-set Sources (no self-source) are public connector data and +// survive replay byte-for-byte. +package sourcecache + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "time" +) + +// RowKind partitions source-cache scopes by the row type they produce. +// It doubles as the row_kind value stored in SourceCacheEntryRecord. +type RowKind string + +const ( + RowKindResources RowKind = "resources" + RowKindEntitlements RowKind = "entitlements" + RowKindGrants RowKind = "grants" +) + +// Valid reports whether k is one of the defined row kinds. +func (k RowKind) Valid() bool { + switch k { + case RowKindResources, RowKindEntitlements, RowKindGrants: + return true + } + return false +} + +// ValidateRowKind returns an error if rowKind is not one of the known +// RowKind* constants. +func ValidateRowKind(rowKind RowKind) error { + if !rowKind.Valid() { + return fmt.Errorf("invalid source cache row kind: %q", rowKind) + } + return nil +} + +// maxScopeKeyLen bounds scope identifiers on the wire and in storage +// keys. Deliberately generous: the shape is a connector convention +// (HashScope produces 64 hex chars) and is not enforced beyond +// non-emptiness and this cap while the model is being proven out against +// real providers. +const maxScopeKeyLen = 256 + +// ValidateScopeKey returns an error when scopeKey is empty or +// unreasonably long. Connectors conventionally use HashScope, but any +// stable identifier is accepted. +func ValidateScopeKey(scopeKey string) error { + if scopeKey == "" { + return fmt.Errorf("source cache scope key is required") + } + if len(scopeKey) > maxScopeKeyLen { + return fmt.Errorf("source cache scope key too long: %d bytes (max %d)", len(scopeKey), maxScopeKeyLen) + } + return nil +} + +// Entry is a previous sync's persisted validator for one scope. +type Entry struct { + // CacheValidator is opaque to the SDK: an HTTP ETag, delta token, etc. + CacheValidator string + + // DiscoveredAt is when the entry was written. + DiscoveredAt time.Time +} + +// Lookup resolves a scope's previous-sync validator. The SDK provides an +// implementation on SyncOpAttrs; connectors call it before revalidating +// upstream. +type Lookup interface { + // LookupPreviousSourceCache returns the previous sync's entry for + // (rowKind, scopeKey). found=false means no entry: fetch fresh. + // Implementations must treat internal read errors that leave fresh + // fetch available as misses rather than failing the connector call. + LookupPreviousSourceCache(ctx context.Context, rowKind RowKind, scopeKey string) (entry Entry, found bool, err error) +} + +// NoopLookup is the Lookup installed when source cache is disabled or +// degraded. Every lookup misses. +type NoopLookup struct{} + +var _ Lookup = NoopLookup{} + +func (NoopLookup) LookupPreviousSourceCache(context.Context, RowKind, string) (Entry, bool, error) { + return Entry{}, false, nil +} + +// SetLookup is implemented by connector clients/servers that can receive a +// source-cache lookup implementation from the sync runner. The SDK calls +// SetSourceCache(lookup) at the start of each sync and SetSourceCache(nil) +// when the sync ends so a late RPC can't read stale state. +type SetLookup interface { + SetSourceCache(ctx context.Context, lookup Lookup) +} + +// HashScope returns the lowercase-hex sha256 of a canonical scope string. +// Convenience for connectors; any stable identifier is acceptable as a +// scope key (only non-emptiness and a length cap are enforced). +func HashScope(canonicalScope string) string { + sum := sha256.Sum256([]byte(canonicalScope)) + return hex.EncodeToString(sum[:]) +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph.go index 23a9f9c9..cf741dcd 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph.go @@ -2,7 +2,9 @@ package expand import ( "context" + "fmt" "iter" + "slices" "sort" "strings" @@ -132,6 +134,111 @@ func (g *EntitlementGraph) IsExpanded() bool { return true } +// MarkExpansionComplete records that every edge has been evaluated and the +// graph passed cycle detection. Persisted graphs must carry these facts so the +// next expansion can safely treat them as completed bases. +func (g *EntitlementGraph) MarkExpansionComplete() { + for edgeID, edge := range g.Edges { + edge.IsExpanded = true + g.Edges[edgeID] = edge + } + g.HasNoCycles = true + g.Actions = nil +} + +// ValidateCompleted checks the durable facts required before a graph may be +// reused as an incremental-expansion base. +func (g *EntitlementGraph) ValidateCompleted() error { + if g == nil { + return fmt.Errorf("graph is nil") + } + if !g.Loaded { + return fmt.Errorf("graph is not fully loaded") + } + if !g.HasNoCycles { + return fmt.Errorf("graph has not completed cycle detection") + } + if !g.IsExpanded() { + return fmt.Errorf("graph has unexpanded edges") + } + maxNodeID := 0 + for nodeID, node := range g.Nodes { + if nodeID > maxNodeID { + maxNodeID = nodeID + } + if node.Id != nodeID { + return fmt.Errorf("node map key %d does not match node id %d", nodeID, node.Id) + } + for _, entitlementID := range node.EntitlementIDs { + if got, ok := g.EntitlementsToNodes[entitlementID]; !ok || got != nodeID { + return fmt.Errorf("entitlement %q does not map back to node %d", entitlementID, nodeID) + } + } + } + if g.NextNodeID < maxNodeID { + return fmt.Errorf("next node id %d is below existing maximum %d", g.NextNodeID, maxNodeID) + } + for entitlementID, nodeID := range g.EntitlementsToNodes { + node, ok := g.Nodes[nodeID] + if !ok || !slices.Contains(node.EntitlementIDs, entitlementID) { + return fmt.Errorf("entitlement map entry %q points to inconsistent node %d", entitlementID, nodeID) + } + } + maxEdgeID := 0 + for edgeID, edge := range g.Edges { + if edgeID > maxEdgeID { + maxEdgeID = edgeID + } + if edge.EdgeID != edgeID { + return fmt.Errorf("edge map key %d does not match edge id %d", edgeID, edge.EdgeID) + } + if _, ok := g.Nodes[edge.SourceID]; !ok { + return fmt.Errorf("edge %d has missing source node %d", edgeID, edge.SourceID) + } + if _, ok := g.Nodes[edge.DestinationID]; !ok { + return fmt.Errorf("edge %d has missing destination node %d", edgeID, edge.DestinationID) + } + if got := g.SourcesToDestinations[edge.SourceID][edge.DestinationID]; got != edgeID { + return fmt.Errorf("edge %d missing from source adjacency", edgeID) + } + if got := g.DestinationsToSources[edge.DestinationID][edge.SourceID]; got != edgeID { + return fmt.Errorf("edge %d missing from destination adjacency", edgeID) + } + } + if g.NextEdgeID < maxEdgeID { + return fmt.Errorf("next edge id %d is below existing maximum %d", g.NextEdgeID, maxEdgeID) + } + for sourceID, destinations := range g.SourcesToDestinations { + for destinationID, edgeID := range destinations { + edge, ok := g.Edges[edgeID] + if !ok || edge.SourceID != sourceID || edge.DestinationID != destinationID { + return fmt.Errorf("source adjacency %d->%d points to inconsistent edge %d", sourceID, destinationID, edgeID) + } + } + } + for destinationID, sources := range g.DestinationsToSources { + for sourceID, edgeID := range sources { + edge, ok := g.Edges[edgeID] + if !ok || edge.SourceID != sourceID || edge.DestinationID != destinationID { + return fmt.Errorf("destination adjacency %d<-%d points to inconsistent edge %d", destinationID, sourceID, edgeID) + } + } + } + return nil +} + +// HasCollapsedCycles reports whether full expansion collapsed an SCC into a +// multi-entitlement node. Such a graph no longer records its internal edges, +// so it cannot safely detect an edge removal that splits the SCC. +func (g *EntitlementGraph) HasCollapsedCycles() bool { + for _, node := range g.Nodes { + if len(node.EntitlementIDs) > 1 { + return true + } + } + return false +} + // IsEntitlementExpanded returns true if all the outgoing edges for the given entitlement have been expanded. func (g *EntitlementGraph) IsEntitlementExpanded(entitlementID string) bool { node := g.GetNode(entitlementID) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph_blob.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph_blob.go new file mode 100644 index 00000000..7e394fc6 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/graph_blob.go @@ -0,0 +1,91 @@ +package expand + +import ( + "bytes" + "encoding/json" + "fmt" + + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" +) + +// graphBlobEnvelope is the serialized form of the entitlement-graph sidecar +// stored in a c1z (instead of bloating the sync token). SyncID guards +// against reading a graph inherited from a different sync (e.g. a fold-copied +// compaction base). +type graphBlobEnvelope struct { + FormatVersion uint32 `json:"format_version"` + SyncID string `json:"sync_id"` + GrantDigest *c1zstore.GrantGenerationDigest `json:"grant_digest,omitempty"` + Graph *EntitlementGraph `json:"graph"` +} + +const graphBlobFormatVersion uint32 = 2 + +// MarshalGraphBlob serializes a legacy, unbound graph blob for compatibility +// tests. Transient state is stripped first (a reload rebuilds it). +// +// The blob has no grant-generation digest, so sync.GraphFromStore deliberately +// rejects it for incremental reuse. Production persistence must use +// MarshalGraphBlobWithGrantDigest. +func MarshalGraphBlob(syncID string, g *EntitlementGraph) ([]byte, error) { + return marshalGraphBlob(syncID, g, nil) +} + +// MarshalGraphBlobWithGrantDigest binds the graph to the exact sealed grant +// generation. Graph reuse requires this binding. +func MarshalGraphBlobWithGrantDigest(syncID string, g *EntitlementGraph, digest c1zstore.GrantGenerationDigest) ([]byte, error) { + if len(digest.Hash) == 0 || digest.ABIVersion == 0 { + return nil, fmt.Errorf("marshal graph blob: incomplete grant digest") + } + digest.Hash = append([]byte(nil), digest.Hash...) + return marshalGraphBlob(syncID, g, &digest) +} + +func marshalGraphBlob(syncID string, g *EntitlementGraph, digest *c1zstore.GrantGenerationDigest) ([]byte, error) { + if g == nil { + return nil, fmt.Errorf("marshal graph blob: nil graph") + } + graphCopy := *g + graphCopy.ClearTransientState() + data, err := json.Marshal(graphBlobEnvelope{FormatVersion: graphBlobFormatVersion, SyncID: syncID, GrantDigest: digest, Graph: &graphCopy}) + if err != nil { + return nil, fmt.Errorf("marshal graph blob: %w", err) + } + return data, nil +} + +// UnmarshalGraphBlob parses a graph for compatibility tests while discarding +// its grant-generation binding. Returns (nil, nil) when the blob belongs to a +// different sync than wantSyncID (stale inherited sidecar); pass "" to skip +// the guard. +// +// The returned graph must not drive incremental reuse. Production readers +// must use UnmarshalGraphBlobWithGrantDigest and verify the returned digest. +func UnmarshalGraphBlob(data []byte, wantSyncID string) (*EntitlementGraph, error) { + graph, _, err := UnmarshalGraphBlobWithGrantDigest(data, wantSyncID) + return graph, err +} + +// UnmarshalGraphBlobWithGrantDigest returns the persisted grant-generation +// binding along with the graph. A nil digest means the blob is unbound and +// must not be reused incrementally. +func UnmarshalGraphBlobWithGrantDigest(data []byte, wantSyncID string) (*EntitlementGraph, *c1zstore.GrantGenerationDigest, error) { + var env graphBlobEnvelope + if err := json.Unmarshal(data, &env); err != nil { + return nil, nil, fmt.Errorf("unmarshal graph blob: %w", err) + } + if env.FormatVersion != graphBlobFormatVersion { + return nil, nil, nil + } + if wantSyncID != "" && env.SyncID != wantSyncID { + return nil, nil, nil + } + if env.Graph == nil { + return nil, nil, nil + } + env.Graph.reinitMaps() + if env.GrantDigest != nil { + env.GrantDigest.Hash = bytes.Clone(env.GrantDigest.Hash) + } + return env.Graph, env.GrantDigest, nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/incremental.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/incremental.go new file mode 100644 index 00000000..f5db117c --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/incremental.go @@ -0,0 +1,590 @@ +package expand + +import ( + "context" + "errors" + "fmt" + "sort" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// ClearTransientState drops the graph's expansion working state — the action +// queue, the projection plan, and metrics — which a persisted graph doesn't +// need for a later reload and which can bloat the sync token. The structural +// graph (nodes, edges, mappings) is untouched. +func (g *EntitlementGraph) ClearTransientState() { + g.Actions = nil + g.ExpansionPlan = nil + g.ExpansionMetrics = nil +} + +// Clone returns a structural deep copy of the graph. Incremental expansion +// mutates the graph, so callers must not share maps or slices with the base. +// This deliberately avoids a JSON round trip: graph cloning is paid on every +// eligible incremental attempt and benchmark evidence showed serialization +// dominated both CPU and allocation at whale scale. +func (g *EntitlementGraph) Clone() (*EntitlementGraph, error) { + if g == nil { + return nil, fmt.Errorf("clone entitlement graph: nil graph") + } + out := &EntitlementGraph{ + NextNodeID: g.NextNodeID, + NextEdgeID: g.NextEdgeID, + Nodes: make(map[int]Node, len(g.Nodes)), + EntitlementsToNodes: make(map[string]int, len(g.EntitlementsToNodes)), + SourcesToDestinations: cloneNestedIntMap(g.SourcesToDestinations), + DestinationsToSources: cloneNestedIntMap(g.DestinationsToSources), + Edges: make(map[int]Edge, len(g.Edges)), + Loaded: g.Loaded, + Depth: g.Depth, + HasNoCycles: g.HasNoCycles, + } + for id, node := range g.Nodes { + node.EntitlementIDs = append([]string(nil), node.EntitlementIDs...) + out.Nodes[id] = node + } + for entitlementID, nodeID := range g.EntitlementsToNodes { + out.EntitlementsToNodes[entitlementID] = nodeID + } + for id, edge := range g.Edges { + edge.ResourceTypeIDs = append([]string(nil), edge.ResourceTypeIDs...) + out.Edges[id] = edge + } + out.Actions = make([]*EntitlementGraphAction, len(g.Actions)) + for i, action := range g.Actions { + if action == nil { + continue + } + actionCopy := *action + actionCopy.Descendants = append([]ActionDescendant(nil), action.Descendants...) + actionCopy.ResourceTypeIDs = append([]string(nil), action.ResourceTypeIDs...) + out.Actions[i] = &actionCopy + } + if g.ExpansionPlan != nil { + plan := *g.ExpansionPlan + plan.Order = append([]int(nil), g.ExpansionPlan.Order...) + plan.ProjectionSources = append([]string(nil), g.ExpansionPlan.ProjectionSources...) + out.ExpansionPlan = &plan + } + if g.ExpansionMetrics != nil { + metrics := *g.ExpansionMetrics + out.ExpansionMetrics = &metrics + } + return out, nil +} + +func cloneNestedIntMap(source map[int]map[int]int) map[int]map[int]int { + out := make(map[int]map[int]int, len(source)) + for outer, inner := range source { + innerCopy := make(map[int]int, len(inner)) + for key, value := range inner { + innerCopy[key] = value + } + out[outer] = innerCopy + } + return out +} + +// reinitMaps replaces nil maps (json leaves absent maps nil) so a +// deserialized graph is immediately usable. +func (g *EntitlementGraph) reinitMaps() { + if g.Nodes == nil { + g.Nodes = map[int]Node{} + } + if g.EntitlementsToNodes == nil { + g.EntitlementsToNodes = map[string]int{} + } + if g.SourcesToDestinations == nil { + g.SourcesToDestinations = map[int]map[int]int{} + } + if g.DestinationsToSources == nil { + g.DestinationsToSources = map[int]map[int]int{} + } + if g.Edges == nil { + g.Edges = map[int]Edge{} + } +} + +// ErrIncrementalFallback means a new edge closed a cycle; the caller should +// re-run a full expansion, which handles cycles correctly. +var ErrIncrementalFallback = errors.New("incremental expansion: change introduces a cycle, fall back to full expansion") + +// ErrIncrementalRevocationDecline means the change is revocation-shaped (an +// existing edge's spec narrowed — shallow-ified, filter tightened, or a source +// dropped), which incremental expansion cannot apply without removing grants. +// The caller declines to full expansion. This is the named hook a future +// tombstone/deletion stage flips from "decline" to "apply deletions". +var ErrIncrementalRevocationDecline = errors.New("incremental expansion: revocation-shaped change, fall back to full expansion") + +// ErrIncrementalDenseChangeDecline means the affected closure is large enough +// that normal full expansion is the safer bounded-cost path. +var ErrIncrementalDenseChangeDecline = errors.New("incremental expansion: dense affected graph, fall back to full expansion") + +const ( + incrementalDenseGraphMinNodes = 1000 + incrementalMaxAffectedPercent = 10 +) + +// NewEdge is one edge to fold in: members of Source also get Destination. +type NewEdge struct { + SourceEntitlementID string + DestEntitlementID string + Shallow bool + ResourceTypeIDs []string +} + +// IncrementalResult reports the impacted subgraph and how many grants were written. +type IncrementalResult struct { + EntitlementsWalked []string + GrantsWritten int +} + +// IncrementalExpander folds new edges into an already-expanded graph and +// propagates the change to only the affected subgraph, reading and writing +// through the same ExpanderStore as the full expander. +// +// Preconditions: graph is a prior completed expansion's graph (edges already +// expanded), and store holds that expansion's grants. Additions only; a new +// edge that closes a cycle returns ErrIncrementalFallback. +type IncrementalExpander struct { + store ExpanderStore + graph *EntitlementGraph + entitlementCache map[string]*v2.Entitlement +} + +func NewIncrementalExpander(store ExpanderStore, graph *EntitlementGraph) *IncrementalExpander { + return &IncrementalExpander{ + store: store, + graph: graph, + entitlementCache: make(map[string]*v2.Entitlement), + } +} + +// ExpandChanges recomputes grants for only the subgraph affected by a set of +// changes. Both kinds seed the walk: newEdges (new expandable relationships, +// added to the graph here) via their destinations, and changedEntitlementIDs +// (entitlements whose membership changed) via their own node. The second kind +// is essential — a membership change adds no edge, so seeding only from +// newEdges would silently drop it. +// +// changedEntitlementIDs is direction-neutral: it names entitlements whose +// membership changed in EITHER direction (added or removed). Whether a removal +// is actually applied is a WRITE-behavior concern, not a seed concern — today +// this method only adds grants (never removes), so callers decline +// revocation-shaped changes to full expansion. When a future stage learns to +// apply deletions, removed-membership entitlements flow through this same +// parameter with no signature change. +// +// The walk reads current membership from the store, so changed members +// (already merged in) propagate without being passed in. Returns +// ErrIncrementalFallback if a new edge closes a cycle. +func (ie *IncrementalExpander) ExpandChanges(ctx context.Context, newEdges []NewEdge, changedEntitlementIDs []string) (*IncrementalResult, error) { + if len(newEdges) == 0 && len(changedEntitlementIDs) == 0 { + return &IncrementalResult{}, nil + } + + seeds := make(map[int]struct{}) + for _, e := range newEdges { + ie.graph.AddEntitlementID(e.SourceEntitlementID) + ie.graph.AddEntitlementID(e.DestEntitlementID) + if err := ie.graph.AddEdge(ctx, e.SourceEntitlementID, e.DestEntitlementID, e.Shallow, e.ResourceTypeIDs); err != nil { + return nil, fmt.Errorf("incremental expansion: add edge %s->%s: %w", e.SourceEntitlementID, e.DestEntitlementID, err) + } + if dst := ie.graph.GetNode(e.DestEntitlementID); dst != nil { + seeds[dst.Id] = struct{}{} + } + } + + // A changed entitlement seeds its own node so descendants are recomputed. + // Entitlements not in the graph have nothing downstream — safely ignored. + for _, entitlementID := range changedEntitlementIDs { + if n := ie.graph.GetNode(entitlementID); n != nil { + seeds[n.Id] = struct{}{} + } + } + + if len(seeds) == 0 { + return &IncrementalResult{}, nil + } + + if cyclic, _ := ie.graph.ComputeCyclicComponents(ctx); len(cyclic) > 0 { + return nil, ErrIncrementalFallback + } + + // Only nodes forward-reachable from a seed are touched. + affected := ie.forwardReachable(seeds) + if len(ie.graph.Nodes) >= incrementalDenseGraphMinNodes && + len(affected)*100 > len(ie.graph.Nodes)*incrementalMaxAffectedPercent { + return nil, ErrIncrementalDenseChangeDecline + } + + // Topological order only the affected closure. Parents outside this set + // were finalized by the base expansion and are read from the store; sorting + // the untouched graph made K=1 work scale with total graph size. + order, err := topologicalAffectedNodeOrder(ie.graph, affected) + if err != nil { + return nil, fmt.Errorf("incremental expansion: topological order: %w", err) + } + + result := &IncrementalResult{} + for _, nodeID := range order { + if err := ctx.Err(); err != nil { + return nil, err // cancelled / run-duration exceeded + } + if _, ok := affected[nodeID]; !ok { + continue + } + node, ok := ie.graph.Nodes[nodeID] + if !ok { + continue + } + for _, destEntitlementID := range node.EntitlementIDs { + written, err := ie.recomputeDestination(ctx, nodeID, destEntitlementID) + if err != nil { + return nil, err + } + result.EntitlementsWalked = append(result.EntitlementsWalked, destEntitlementID) + result.GrantsWritten += written + } + } + ie.graph.MarkExpansionComplete() + return result, nil +} + +func topologicalAffectedNodeOrder(g *EntitlementGraph, affected map[int]struct{}) ([]int, error) { + inDegree := make(map[int]int, len(affected)) + for nodeID := range affected { + if _, ok := g.Nodes[nodeID]; ok { + inDegree[nodeID] = 0 + } + } + for sourceID := range inDegree { + for destinationID := range g.SourcesToDestinations[sourceID] { + if _, ok := inDegree[destinationID]; ok { + inDegree[destinationID]++ + } + } + } + frontier := make(intMinHeap, 0, len(inDegree)) + for nodeID, degree := range inDegree { + if degree == 0 { + frontier.push(nodeID) + } + } + order := make([]int, 0, len(inDegree)) + for len(frontier) > 0 { + nodeID := frontier.pop() + order = append(order, nodeID) + for childID := range g.SourcesToDestinations[nodeID] { + if _, ok := inDegree[childID]; !ok { + continue + } + inDegree[childID]-- + if inDegree[childID] == 0 { + frontier.push(childID) + } + } + } + if len(order) != len(inDegree) { + return nil, fmt.Errorf("incremental expansion: affected graph contains a cycle or dangling edge") + } + return order, nil +} + +// intMinHeap keeps the smallest ready node at the front without re-sorting +// every ready node after each insertion. +type intMinHeap []int + +func (h *intMinHeap) push(nodeID int) { + *h = append(*h, nodeID) + for child := len(*h) - 1; child > 0; { + parent := (child - 1) / 2 + if (*h)[parent] <= (*h)[child] { + break + } + (*h)[parent], (*h)[child] = (*h)[child], (*h)[parent] + child = parent + } +} + +func (h *intMinHeap) pop() int { + root := (*h)[0] + last := len(*h) - 1 + (*h)[0] = (*h)[last] + *h = (*h)[:last] + + for parent := 0; ; { + left := 2*parent + 1 + if left >= len(*h) { + break + } + child := left + right := left + 1 + if right < len(*h) && (*h)[right] < (*h)[left] { + child = right + } + if (*h)[parent] <= (*h)[child] { + break + } + (*h)[parent], (*h)[child] = (*h)[child], (*h)[parent] + parent = child + } + + return root +} + +func (ie *IncrementalExpander) forwardReachable(seeds map[int]struct{}) map[int]struct{} { + reached := make(map[int]struct{}) + queue := make([]int, 0, len(seeds)) + for id := range seeds { + reached[id] = struct{}{} + queue = append(queue, id) + } + for len(queue) > 0 { + cur := queue[0] + queue = queue[1:] + for child := range ie.graph.SourcesToDestinations[cur] { + if _, ok := reached[child]; !ok { + reached[child] = struct{}{} + queue = append(queue, child) + } + } + } + return reached +} + +// incrementalFlushChunk caps buffered new grants before a flush, so a whale +// destination doesn't materialize its whole output. Mirrors the full +// expander's expansionDirtyFlushChunk. A var only so tests can lower it. +var incrementalFlushChunk = 10000 + +// recomputeDestination writes destEntitlementID's implied grants that aren't +// already present, returning how many. Source grants stream a page at a time +// and writes flush in chunks, so peak memory is one page + one flush buffer + +// the destination's existing-key set — not the whole source or output. +func (ie *IncrementalExpander) recomputeDestination(ctx context.Context, nodeID int, destEntitlementID string) (int, error) { + destEnt, err := ie.getEntitlement(ctx, destEntitlementID) + if err != nil { + return 0, err + } + if destEnt == nil { + // Dangling ref: skip-with-warn, matching the full evaluator (don't + // error into a fallback). + ctxzap.Extract(ctx).Warn("incremental expansion: destination entitlement not in store; skipping", + zap.String("entitlement_id", destEntitlementID)) + return 0, nil + } + + // 1. Accumulate, per principal, the union of sources contributed by all + // incoming edges. Streaming reads keep only one source page live, but the + // contribution map holds one entry per distinct principal across ALL + // sources feeding this destination — the same worst-case fan-in footprint + // the full expander's per-destination reduce carries. + contrib := make(map[string]*principalContribution) + for sourceNodeID, edgeID := range ie.graph.DestinationsToSources[nodeID] { + edge, ok := ie.graph.Edges[edgeID] + if !ok { + continue + } + sourceNode, ok := ie.graph.Nodes[sourceNodeID] + if !ok { + continue + } + for _, sourceEntitlementID := range sourceNode.EntitlementIDs { + // The store's read path requires a full entitlement record (with + // resource refs), not a bare id — fetch it. A dangling ref (source + // not in the store) is skipped-with-warn, matching the full evaluator. + sourceEnt, err := ie.getEntitlement(ctx, sourceEntitlementID) + if err != nil { + return 0, err + } + if sourceEnt == nil { + ctxzap.Extract(ctx).Warn("incremental expansion: source entitlement not in store; skipping", + zap.String("entitlement_id", sourceEntitlementID)) + continue + } + perGrantErr := ie.forEachGrant(ctx, sourceEnt, edge.ResourceTypeIDs, func(sourceGrant *v2.Grant) error { + // Shared definition of "contributes" with the full expander: + // rejects nil-principal grants, off-type principals, and + // non-direct grants over shallow edges. + if !grantContributesOverEdge(sourceGrant, sourceEntitlementID, edge) { + return nil + } + // Directness is relative to the source entitlement (matches the + // full expander): a plain direct grant or one whose sources map + // records this entitlement counts as direct. + isSourceDirect := isGrantDirectOnEntitlement(sourceGrant, sourceEntitlementID) + principal := sourceGrant.GetPrincipal() + pid := principal.GetId() + key := pid.GetResourceType() + "\x00" + pid.GetResource() + pc := contrib[key] + if pc == nil { + pc = &principalContribution{principal: principal} + contrib[key] = pc + } + pc.addSource(sourceEntitlementID, isSourceDirect) + return nil + }) + if perGrantErr != nil { + return 0, perGrantErr + } + } + } + if len(contrib) == 0 { + return 0, nil + } + + buf := make([]*v2.Grant, 0, incrementalFlushChunk) + written := 0 + flush := func() error { + if len(buf) == 0 { + return nil + } + if err := ie.store.StoreExpandedGrants(ctx, buf...); err != nil { + return fmt.Errorf("incremental expansion: store grants on %s: %w", destEntitlementID, err) + } + written += len(buf) + buf = buf[:0] + return nil + } + + // 2. Merge contributions into the destination's existing grants (union the + // sources map, upgrade direct-ness), streaming one page at a time. Only a + // grant that actually changed is rewritten. A principal can hold several + // grant rows on one entitlement (connector-authored IDs are arbitrary), and + // the full expander merges the contribution into every row sharing the + // principal key — so record matches in a side set instead of consuming the + // contribution on the first row, and drop them from contrib only after the + // whole destination has streamed. + matched := make(map[string]struct{}) + mergeErr := ie.forEachGrant(ctx, destEnt, nil, func(g *v2.Grant) error { + pid := g.GetPrincipal().GetId() + key := pid.GetResourceType() + "\x00" + pid.GetResource() + pc := contrib[key] + if pc == nil { + return nil + } + matched[key] = struct{}{} + updated := mergeContributionIntoExistingGrant(g, destEntitlementID, pc.sources) + if updated == nil { + return nil // already had these sources — no write + } + buf = append(buf, updated) + if len(buf) >= incrementalFlushChunk { + return flush() + } + return nil + }) + if mergeErr != nil { + return 0, mergeErr + } + for key := range matched { + delete(contrib, key) + } + + // 3. Whatever is left in contrib are brand-new principals. Sort for + // deterministic (byte-stable) output. + newKeys := make([]string, 0, len(contrib)) + for key := range contrib { + newKeys = append(newKeys, key) + } + sort.Strings(newKeys) + for _, key := range newKeys { + pc := contrib[key] + grant, err := newExpandedGrantWithSources(destEnt, pc.principal, pc.sources) + if err != nil { + return 0, fmt.Errorf("incremental expansion: build grant on %s: %w", destEntitlementID, err) + } + buf = append(buf, grant) + if len(buf) >= incrementalFlushChunk { + if err := flush(); err != nil { + return 0, err + } + } + } + + if err := flush(); err != nil { + return 0, err + } + return written, nil +} + +// principalContribution accumulates the source entitlements contributing one +// principal to a destination. sources is a small slice (fan-in is tiny), deduped +// by entitlement id with direct-ness upgraded to true if any contribution is direct. +type principalContribution struct { + principal *v2.Resource + sources batonGrant.Sources +} + +func (pc *principalContribution) addSource(entitlementID string, isDirect bool) { + for i := range pc.sources { + if pc.sources[i].EntitlementID == entitlementID { + if isDirect && !pc.sources[i].IsDirect { + pc.sources[i].IsDirect = true + } + return + } + } + pc.sources = append(pc.sources, batonGrant.Source{EntitlementID: entitlementID, IsDirect: isDirect}) +} + +// getEntitlement fetches an entitlement, returning (nil, nil) for a dangling +// ref (NotFound) so callers skip it — matching the full evaluator, which treats +// NotFound as skip rather than a hard error. +func (ie *IncrementalExpander) getEntitlement(ctx context.Context, entitlementID string) (*v2.Entitlement, error) { + if entitlement, ok := ie.entitlementCache[entitlementID]; ok { + return entitlement, nil + } + resp, err := ie.store.GetEntitlement(ctx, reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ + EntitlementId: entitlementID, + }.Build()) + if err != nil { + if status.Code(err) == codes.NotFound { + ie.entitlementCache[entitlementID] = nil + return nil, nil + } + return nil, fmt.Errorf("incremental expansion: get entitlement %s: %w", entitlementID, err) + } + if resp == nil { + ie.entitlementCache[entitlementID] = nil + return nil, nil + } + entitlement := resp.GetEntitlement() + ie.entitlementCache[entitlementID] = entitlement + return entitlement, nil +} + +// forEachGrant streams an entitlement's grants (filtered by resourceTypeIDs) +// one page at a time, invoking fn per grant — never materializing the whole +// set. entitlement must be a full record (with resource refs); the store's read +// path rejects bare-id entitlements. +func (ie *IncrementalExpander) forEachGrant(ctx context.Context, entitlement *v2.Entitlement, resourceTypeIDs []string, fn func(*v2.Grant) error) error { + pageToken := "" + for { + resp, err := ie.store.ListGrantsForEntitlement(ctx, reader_v2.GrantsReaderServiceListGrantsForEntitlementRequest_builder{ + Entitlement: entitlement, + PrincipalResourceTypeIds: resourceTypeIDs, + PageToken: pageToken, + }.Build()) + if err != nil { + return fmt.Errorf("incremental expansion: list grants for %s: %w", entitlement.GetId(), err) + } + for _, g := range resp.GetList() { + if err := fn(g); err != nil { + return err + } + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + return nil + } + } +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge.go index c3b35807..37eee020 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/expand/topological_merge.go @@ -429,11 +429,7 @@ func (e *Expander) driveTopologicalLayer( // queue. The topological evaluators expand the whole graph in one pass, so they // finalize all edges together at the end rather than per action. func (e *Expander) markExpansionComplete() { - for edgeID, edge := range e.graph.Edges { - edge.IsExpanded = true - e.graph.Edges[edgeID] = edge - } - e.graph.Actions = nil + e.graph.MarkExpansionComplete() } func sortedCopy(in []string) []string { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/external_principal_index.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/external_principal_index.go index 06792e02..22f4de78 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/external_principal_index.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/external_principal_index.go @@ -190,7 +190,7 @@ func foldKey(s string) string { var b strings.Builder b.Grow(len(s)) for _, r := range s { - b.WriteRune(foldRune(r)) + _, _ = b.WriteRune(foldRune(r)) } return b.String() } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/ingest_invariants.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/ingest_invariants.go index 5a278504..3bb9efbe 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/ingest_invariants.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/ingest_invariants.go @@ -163,8 +163,7 @@ func invariantVerdict(err error) error { return &invariantVerdictError{err: err} // TypeScopedGrants) land, they register here against I7 and I8 // respectively — type-granularity scopes are exactly the shapes those // referential checks exist for. -// -//nolint:gosec // G101 false positive: "PageTokens" is an annotation name, not a credential. +// #nosec G101 -- "PageTokens" is an annotation name, not a credential. var sideEffectAnnotationCoverage = map[string]string{ "c1.connector.v2.GrantExpandable": "I1: response-loop expansion arming (SetNeedsExpansion) + needs_expansion column persistence; store-derived probe arrives with replay", "c1.connector.v2.ExternalResourceMatch": "I2: response-loop match arming (SetHasExternalResourcesGrants); store-derived existence-bit repair arrives with replay", @@ -449,10 +448,43 @@ func ingestInvariantHaltStages() []string { // store-level function so store-producing pipelines without a syncer // (the compactor's expand pass) can enforce the same contract. func RunIngestInvariants(ctx context.Context, store connectorstore.Reader, policy IngestInvariantsPolicy) error { - _, err := runIngestInvariants(ctx, store, policy) + _, err := RunIngestInvariantsWithVerification(ctx, store, policy) return err } +// RunIngestInvariantsWithVerification evaluates the invariant pass and returns +// the verification metadata a store-producing caller must persist after the +// sync is sealed. It does not write the marker itself: publishing proof before +// EndSync would allow an unfinished artifact to claim verification. +func RunIngestInvariantsWithVerification( + ctx context.Context, + store connectorstore.Reader, + policy IngestInvariantsPolicy, +) (*c1zstore.IngestInvariantVerification, error) { + coverage, err := runIngestInvariants(ctx, store, policy) + if err != nil { + return nil, err + } + return &c1zstore.IngestInvariantVerification{ + Generation: IngestInvariantGeneration, + Coverage: coverage, + Mode: ingestInvariantVerificationMode(policy), + }, nil +} + +func ingestInvariantVerificationMode(policy IngestInvariantsPolicy) c1zstore.IngestInvariantVerificationMode { + switch { + case policy.CompactionMerge && policy.FailFast: + return c1zstore.IngestInvariantVerificationModeCompactionMergeFailFast + case policy.CompactionMerge: + return c1zstore.IngestInvariantVerificationModeCompactionMerge + case policy.FailFast: + return c1zstore.IngestInvariantVerificationModeConnectorFailFast + default: + return c1zstore.IngestInvariantVerificationModeConnector + } +} + // runIngestInvariants returns the IDs of checks that actually completed. The // public wrapper intentionally retains its existing error-only API; the syncer // consumes the coverage to persist verification provenance. @@ -608,24 +640,11 @@ func (s *syncer) runIngestionInvariants(ctx context.Context) error { if s.testIngestHaltHook != nil { policy.halt = s.testIngestHaltHook } - coverage, err := runIngestInvariants(ctx, s.store, policy) + verification, err := RunIngestInvariantsWithVerification(ctx, s.store, policy) if err != nil { return err } - mode := c1zstore.IngestInvariantVerificationModeConnector - switch { - case policy.CompactionMerge && policy.FailFast: - mode = c1zstore.IngestInvariantVerificationModeCompactionMergeFailFast - case policy.CompactionMerge: - mode = c1zstore.IngestInvariantVerificationModeCompactionMerge - case policy.FailFast: - mode = c1zstore.IngestInvariantVerificationModeConnectorFailFast - } - s.pendingInvariantVerification = &c1zstore.IngestInvariantVerification{ - Generation: IngestInvariantGeneration, - Coverage: coverage, - Mode: mode, - } + s.pendingInvariantVerification = verification return nil } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/parallel_syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/parallel_syncer.go index cc7111d4..35cbd737 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/parallel_syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/parallel_syncer.go @@ -400,9 +400,11 @@ func (s *syncer) parallelSync( continue case SyncGrantExpansionOp: - // Mark the sync as supporting diff, but only if we're starting fresh. - // If we're resuming (graph has edges or a page token), we may be continuing - // from old code that didn't have this marker, so we must not set it. + // Stamp the supports_diff marker (data collection complete; the + // name is historical — it gates `baton rollback-expansion`), but + // only if we're starting fresh. If we're resuming (graph has edges + // or a page token), we may be continuing from old code that didn't + // have this marker, so we must not set it. entitlementGraph := s.state.EntitlementGraph(ctx) isResumingExpansion := entitlementGraph.Loaded || len(entitlementGraph.Edges) > 0 || stateAction.PageToken != "" if !isResumingExpansion { @@ -411,8 +413,8 @@ func (s *syncer) parallelSync( } if err := s.store.SyncMeta().MarkSyncSupportsDiff(ctx, s.syncID); err != nil { // No detached rescue on this exit (RFC 0009 §4.2): a - // metadata-only write for the unused diff-sync feature, - // with no progress since the loop-top checkpoint. + // metadata-only write, with no progress since the + // loop-top checkpoint. l.Error("failed to set supports_diff marker", zap.Error(err)) return warnings, err } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go index 9fbd487e..ecec42c4 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/state.go @@ -1,6 +1,7 @@ package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility import ( + "bytes" "context" "encoding/json" "errors" @@ -10,6 +11,7 @@ import ( "sync" "time" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/sync/expand" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" @@ -34,7 +36,9 @@ type State interface { FinishAction(ctx context.Context, action *Action) NextPage(ctx context.Context, actionID string, pageToken string) error EntitlementGraph(ctx context.Context) *expand.EntitlementGraph + PeekEntitlementGraph() *expand.EntitlementGraph ClearEntitlementGraph(ctx context.Context) + ClearEntitlementGraphTransientState(ctx context.Context) Current() *Action GetAction(id string) *Action PeekMatchingActions(ctx context.Context, op ActionOp) []*Action @@ -86,6 +90,12 @@ func PrepareExpansionReplayToken(stateStr string) (string, error) { return "", err } st.SetNeedsExpansion() + // Clear any preserved entitlement graph. A graph preserved by + // WithPreserveEntitlementGraph has Loaded=true with every edge already + // marked expanded, so a replayed sync would skip graph loading and the + // expander would report done immediately — the replay would silently + // no-op. Clearing it makes the replay rebuild the graph from scratch. + st.ClearEntitlementGraph(context.Background()) if st.Current() == nil { // A finished sync deserializes with no action map, so seed one before // queuing the InitOp that drives the resumed run. @@ -97,6 +107,68 @@ func PrepareExpansionReplayToken(stateStr string) (string, error) { return st.Marshal() } +// GraphFromToken parses a legacy sync token and returns its entitlement graph +// for compatibility tests. It returns nil if the token carried no graph. +// +// Token graphs have no grant-generation binding and must not drive incremental +// reuse. Production readers must use GraphFromStore, which verifies that the +// sidecar graph describes the store's exact sealed grant generation. +func GraphFromToken(stateStr string) (*expand.EntitlementGraph, error) { + st := newState() + if err := st.Unmarshal(stateStr); err != nil { + return nil, err + } + return st.entitlementGraph, nil +} + +// EntitlementGraphStore is the optional store capability backing graph +// persistence in the c1z (Pebble implements it; SQLite does not). The blob +// format is owned by pkg/sync/expand. +type EntitlementGraphStore interface { + PutEntitlementGraphBlob(ctx context.Context, data []byte) error + GetEntitlementGraphBlob(ctx context.Context) ([]byte, error) + DeleteEntitlementGraphBlob(ctx context.Context) error +} + +// GraphFromStore loads the entitlement graph persisted in the c1z sidecar for +// syncID. Returns nil (no error) when the store lacks the capability, no graph +// was preserved, or the stored graph belongs to a different sync. +func GraphFromStore(ctx context.Context, store c1zstore.Store, syncID string) (*expand.EntitlementGraph, error) { + gs, ok := store.(EntitlementGraphStore) + if !ok { + return nil, nil + } + data, err := gs.GetEntitlementGraphBlob(ctx) + if err != nil { + return nil, err + } + if data == nil { + return nil, nil + } + graph, boundDigest, err := expand.UnmarshalGraphBlobWithGrantDigest(data, syncID) + if err != nil || graph == nil { + return graph, err + } + if boundDigest == nil { + return nil, nil + } + digestReader, ok := store.(c1zstore.GrantGenerationDigestReader) + if !ok { + return nil, nil + } + currentDigest, found, err := digestReader.GrantGenerationDigest(ctx) + if err != nil { + return nil, err + } + if !found || + boundDigest.Count != currentDigest.Count || + boundDigest.ABIVersion != currentDigest.ABIVersion || + !bytes.Equal(boundDigest.Hash, currentDigest.Hash) { + return nil, nil + } + return graph, nil +} + // ActionOp represents a sync operation. type ActionOp uint8 @@ -1093,11 +1165,28 @@ func (st *state) EntitlementGraph(ctx context.Context) *expand.EntitlementGraph return st.entitlementGraph } +// PeekEntitlementGraph returns the graph without allocating one when absent +// (unlike EntitlementGraph). Used by the preserve path to decide whether +// there is a graph worth persisting. +func (st *state) PeekEntitlementGraph() *expand.EntitlementGraph { + return st.entitlementGraph +} + // ClearEntitlementGraph clears the entitlement graph. This is meant to make the final sync token less confusing. func (st *state) ClearEntitlementGraph(ctx context.Context) { st.entitlementGraph = nil } +// ClearEntitlementGraphTransientState strips a preserved graph's expansion +// working state before the final checkpoint. A no-op when no graph was built — +// deliberately NOT EntitlementGraph(ctx), which would allocate an empty graph +// into the final token where prior behavior serialized none. +func (st *state) ClearEntitlementGraphTransientState(_ context.Context) { + if st.entitlementGraph != nil { + st.entitlementGraph.ClearTransientState() + } +} + func (st *state) GetCompletedActionsCount() uint64 { st.mtx.RLock() defer st.mtx.RUnlock() diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go index 516d3669..cadd2fef 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/sync/syncer.go @@ -20,6 +20,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/bid" "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + enginepkg "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" "github.com/conductorone/baton-sdk/pkg/sync/expand" "github.com/conductorone/baton-sdk/pkg/types/entitlement" batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" @@ -206,19 +207,20 @@ type syncer struct { // event (seed/dequeue/commit/abort/done) for post-hoc verification // of the queue contract. Nil in production: one pointer check per // queue operation. - testQueueAudit *queueAudit - connector types.ConnectorClient - state State - runDuration time.Duration - transitionHandler func(s Action) - progressHandler func(p *Progress) - tmpDir string - storageEngine c1zstore.Engine - skipFullSync bool - lastCheckPointTime time.Time - counts *progresslog.ProgressLog - targetedSyncResources []*v2.Resource - onlyExpandGrants bool + testQueueAudit *queueAudit + connector types.ConnectorClient + state State + runDuration time.Duration + transitionHandler func(s Action) + progressHandler func(p *Progress) + tmpDir string + storageEngine c1zstore.Engine + skipFullSync bool + lastCheckPointTime time.Time + counts *progresslog.ProgressLog + targetedSyncResources []*v2.Resource + onlyExpandGrants bool + preserveEntitlementGraph bool // compactionMergedStore marks the store as a pre-sealed artifact // this process did not collect (WithCompactionMergedStore — the // compactor's keep-newer merge and rollback-expansion's replay): @@ -270,6 +272,44 @@ type expanderStoreAdapter struct { store c1zstore.Store } +// NewExpanderStore adapts a c1zstore.Store into an expand.ExpanderStore, +// bridging engine differences (Pebble exposes StoreExpandedGrants on its +// Grants() sub-store, SQLite at top level). Use this instead of type-asserting +// the store, which is unsafe for Pebble. +func NewExpanderStore(store c1zstore.Store) expand.ExpanderStore { + return expanderStoreAdapter{store: store} +} + +// persistEntitlementGraphToStore binds the preserved graph to the exact sealed +// grant generation and writes both into the c1z sidecar. +func (s *syncer) persistEntitlementGraphToStore(ctx context.Context, syncID string, g *expand.EntitlementGraph) { + if g == nil { + return + } + gs, ok := s.store.(EntitlementGraphStore) + if !ok { + return + } + digestReader, ok := s.store.(c1zstore.GrantGenerationDigestReader) + if !ok { + return + } + digest, found, err := digestReader.GrantGenerationDigest(ctx) + if err != nil || !found { + ctxzap.Extract(ctx).Warn("preserve entitlement graph: sealed grant digest unavailable; graph will not be reusable", zap.Error(err)) + return + } + data, err := expand.MarshalGraphBlobWithGrantDigest(syncID, g, digest) + if err != nil { + ctxzap.Extract(ctx).Warn("preserve entitlement graph: marshal failed", zap.Error(err)) + return + } + if err := gs.PutEntitlementGraphBlob(ctx, data); err != nil { + ctxzap.Extract(ctx).Warn("preserve entitlement graph: sidecar write failed", zap.Error(err)) + return + } +} + func (a expanderStoreAdapter) GetEntitlement(ctx context.Context, req *reader_v2.EntitlementsReaderServiceGetEntitlementRequest) (*reader_v2.EntitlementsReaderServiceGetEntitlementResponse, error) { return a.store.GetEntitlement(ctx, req) } @@ -1058,8 +1098,22 @@ func (s *syncer) Sync(ctx context.Context) error { } // Force a checkpoint to clear completed actions & entitlement graph in sync_token. - s.state.ClearEntitlementGraph(ctx) - + // preserveEntitlementGraph keeps the graph for a later incremental + // expansion: written to the c1z sidecar when the store supports it (token + // stays skinny — a whale graph is megabytes), else kept in the final token. + // Transient working state is stripped either way; a reload rebuilds it. + var graphToPersist *expand.EntitlementGraph + if s.preserveEntitlementGraph { + s.state.ClearEntitlementGraphTransientState(ctx) + _, hasGraphSidecar := s.store.(EntitlementGraphStore) + _, hasGrantDigest := s.store.(c1zstore.GrantGenerationDigestReader) + if hasGraphSidecar && hasGrantDigest { + graphToPersist = s.state.PeekEntitlementGraph() + s.state.ClearEntitlementGraph(ctx) + } + } else { + s.state.ClearEntitlementGraph(ctx) + } err = s.Checkpoint(ctx, true) if err != nil { // Deliberately no detached rescue (RFC 0009 §4.2): the plan is @@ -1083,6 +1137,10 @@ func (s *syncer) Sync(ctx context.Context) error { if err != nil { return s.returnSyncError(l, span, err) } + // EndSync built the authoritative whole-file grant digest. Persisting the + // graph now binds it to that exact sealed grant generation. A crash before + // this write leaves no reusable graph and therefore fails safe. + s.persistEntitlementGraphToStore(ctx, syncID, graphToPersist) // The sync is sealed: publish the verification the invariant pass // staged. Marking only after EndSync keeps the marker off unfinished @@ -3967,21 +4025,24 @@ func WithExternalResourceC1ZPath(path string) SyncOpt { } } -// WithPreviousSyncC1ZPath points ETag-replay at a separate c1z holding -// the previous sync, instead of reading a previous sync from inside the -// live store. +// WithPreviousSyncC1ZPath registers a separate c1z holding the previous sync +// for replay features. // // This is required for the single-sync v3 (Pebble) engine: a Pebble c1z // holds exactly one sync by contract, so there is no in-file "previous -// sync" to replay from (StartNewSync replaces the prior sync). Supplying -// the prior run's c1z here lets the syncer recover unchanged resources' -// ETags and carry their grants forward across runs. When unset, replay -// falls back to reading a previous sync from the live store (the SQLite -// multi-sync behavior), so existing callers are unaffected. +// sync" to replay from (StartNewSync replaces the prior sync). NewSyncer +// currently validates and retains the eligible reader as orchestration +// scaffolding; the ETag read/carry-forward path does not yet consume it. +// +// The explicitly named file is strict: open, metadata-read, and close failures +// are returned. A valid file whose run is simply ineligible warns and degrades +// to a cold sync. Use WithOptionalPreviousSyncC1ZPath for cache-style inputs +// where every unusable-file failure must degrade instead. // -// The file is opened read-only and engine-agnostically (the magic byte -// selects SQLite or Pebble), so the previous-sync c1z may use either -// engine. +// The file is opened read-only and engine-agnostically so its format can be +// diagnosed, but source-cache replay currently requires a Pebble artifact. +// SQLite inputs are valid c1z files but are treated as cold inputs because they +// do not carry the replay indexes and compaction provenance this gate requires. func WithPreviousSyncC1ZPath(path string) SyncOpt { return func(s *syncer) { s.previousSyncC1ZPath = path @@ -3996,7 +4057,7 @@ func WithPreviousSyncC1ZPath(path string) SyncOpt { // caller maintains automatically (the service-mode previous-sync spare) // — a bad cache file must never fail a sync. Callers that name a // specific file deliberately should use WithPreviousSyncC1ZPath, which -// surfaces open failures. +// surfaces open, metadata-read, and close failures. func WithOptionalPreviousSyncC1ZPath(path string) SyncOpt { return func(s *syncer) { s.previousSyncC1ZPath = path @@ -4077,6 +4138,15 @@ func WithCompactionMergedStore() SyncOpt { } } +// WithPreserveEntitlementGraph preserves the entitlement graph for later +// incremental expansion. Pebble stores it in the c1z sidecar; stores without +// that capability retain it in the final sync token as a legacy fallback. +func WithPreserveEntitlementGraph() SyncOpt { + return func(s *syncer) { + s.preserveEntitlementGraph = true + } +} + // WithDontExpandGrants sets whether to skip expanding grants. // This is used for speeding up service mode connectors and reducing their c1z upload size. // C1 will process the uploaded c1z and expand grants itself. @@ -4219,14 +4289,63 @@ func NewSyncer(ctx context.Context, c types.ConnectorClient, opts ...SyncOpt) (S if s.previousSyncC1ZPath != "" { // Open the previous-sync c1z read-only and engine-agnostically - // (NewStore selects the engine from the file's magic byte), so a - // Pebble or SQLite prior run both work as a replay source. + // (NewStore selects the engine from the file's magic byte), then + // require Pebble: source-cache manifests and replay indexes are a + // Pebble capability, so SQLite artifacts are cold inputs. previousSyncStore, err := dotc1z.NewStore(ctx, s.previousSyncC1ZPath, dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(s.tmpDir), ) switch { case err == nil: + if _, ok := enginepkg.AsEngine(previousSyncStore); !ok { + if closeErr := previousSyncStore.Close(ctx); closeErr != nil { + if s.previousSyncC1ZPathOptional { + ctxzap.Extract(ctx).Warn("non-Pebble previous-sync c1z could not close cleanly; syncing without source-cache replay", + zap.String("previous_sync_c1z_path", s.previousSyncC1ZPath), + zap.Error(closeErr), + ) + break + } + return nil, fmt.Errorf("error closing non-Pebble previous-sync c1z %q: %w", s.previousSyncC1ZPath, closeErr) + } + ctxzap.Extract(ctx).Warn("previous-sync c1z uses an engine that is not replay-eligible; syncing without source-cache replay", + zap.String("previous_sync_c1z_path", s.previousSyncC1ZPath), + ) + break + } + run, metaErr := previousSyncStore.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + if metaErr != nil { + closeErr := previousSyncStore.Close(ctx) + if s.previousSyncC1ZPathOptional { + ctxzap.Extract(ctx).Warn("previous-sync c1z metadata unusable; syncing without source-cache replay", + zap.String("previous_sync_c1z_path", s.previousSyncC1ZPath), + zap.Error(errors.Join(metaErr, closeErr)), + ) + break + } + return nil, fmt.Errorf( + "error reading previous-sync c1z %q metadata: %w", + s.previousSyncC1ZPath, + errors.Join(metaErr, closeErr), + ) + } + if run == nil || !run.UsableAsReplaySource() { + if closeErr := previousSyncStore.Close(ctx); closeErr != nil { + if s.previousSyncC1ZPathOptional { + ctxzap.Extract(ctx).Warn("ineligible previous-sync c1z could not close cleanly; syncing without source-cache replay", + zap.String("previous_sync_c1z_path", s.previousSyncC1ZPath), + zap.Error(closeErr), + ) + break + } + return nil, fmt.Errorf("error closing ineligible previous-sync c1z %q: %w", s.previousSyncC1ZPath, closeErr) + } + ctxzap.Extract(ctx).Warn("previous-sync c1z is not replay-eligible; syncing without source-cache replay", + zap.String("previous_sync_c1z_path", s.previousSyncC1ZPath), + ) + break + } s.previousSyncReader = previousSyncStore case s.previousSyncC1ZPathOptional: // Best-effort replay source (see WithOptionalPreviousSyncC1ZPath): diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go index 97c1e50b..96e29359 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/attached/attached.go @@ -42,7 +42,6 @@ func NewAttachedCompactor(base, applied c1zstore.Store) (*Compactor, error) { } func latestFinishedCompactableSync(ctx context.Context, f *dotc1z.C1File) (*reader_v2.SyncRun, error) { - // Compaction must NOT operate on diff syncs (partial_upserts / partial_deletions). // We want the latest finished "snapshot-like" sync. candidates := []connectorstore.SyncType{ connectorstore.SyncTypeFull, @@ -82,11 +81,7 @@ func (c *Compactor) Compact(ctx context.Context) error { return fmt.Errorf("failed to get base sync: %w", err) } if baseSync == nil { - return fmt.Errorf( - "no finished compactable sync found in base (diff sync types %q/%q are not compactable)", - string(connectorstore.SyncTypePartialUpserts), - string(connectorstore.SyncTypePartialDeletions), - ) + return fmt.Errorf("no finished compactable sync found in base") } appliedSync, err := latestFinishedCompactableSync(ctx, c.applied) @@ -94,11 +89,7 @@ func (c *Compactor) Compact(ctx context.Context) error { return fmt.Errorf("failed to get applied sync: %w", err) } if appliedSync == nil { - return fmt.Errorf( - "no finished compactable sync found in applied (diff sync types %q/%q are not compactable)", - string(connectorstore.SyncTypePartialUpserts), - string(connectorstore.SyncTypePartialDeletions), - ) + return fmt.Errorf("no finished compactable sync found in applied") } l := ctxzap.Extract(ctx) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go index b1cfeee3..c6f9bdfc 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor.go @@ -8,19 +8,25 @@ import ( "os" "path" "path/filepath" + "sort" "time" + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + enginepkg "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" "github.com/conductorone/baton-sdk/pkg/sdk" "github.com/conductorone/baton-sdk/pkg/sync" + "github.com/conductorone/baton-sdk/pkg/sync/expand" "github.com/conductorone/baton-sdk/pkg/synccompactor/attached" "github.com/conductorone/baton-sdk/pkg/tempdir" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.opentelemetry.io/otel" "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" "github.com/conductorone/baton-sdk/pkg/uotel" ) @@ -44,10 +50,30 @@ type Compactor struct { syncLimit int c1zOptions []dotc1z.C1ZOption skipGrantExpansion bool + failFastInvariants bool + // incrementalExpansion enables diff-aware expansion. The compactor loads the + // graph itself from entries[0], so callers cannot pair a graph with the wrong + // artifact. incrementalBaseGraph holds that validated, store-loaded graph. + // The set of changed entitlements is derived from the applied increments + // during expansion, not supplied by the caller. + incrementalExpansion bool + incrementalBaseGraph *expand.EntitlementGraph + // incrementalExpansionRan records whether the diff-aware path actually + // handled expansion (vs falling back to full). Read by tests to prove the + // fast path ran rather than silently falling back. + incrementalExpansionRan bool + // incrementalTestHook is a package-private fault seam used by crash/retry + // tests. Production compactions leave it nil. + incrementalTestHook func(stage string) error + // foldChangedEntitlementIDs: changed-entitlement set collected by the + // Pebble fold; nil when no fold ran (derive fallback). + foldChangedEntitlementIDs map[string]struct{} // engine selects the storage engine for the compacted output. - // Empty means EngineSQLite (the default; behavior is unchanged and - // the output is byte-identical to the pre-engine-option compactor). - // EnginePebble produces a v3 Pebble c1z via a native record merge. + // Empty means "follow the inputs": Compact resolves it via + // inferEngineFromInputs (any Pebble input → Pebble, all-SQLite → + // SQLite), so the compactor does NOT follow the dotc1z engine + // default. EnginePebble produces a v3 Pebble c1z via a native + // record merge. engine c1zstore.Engine // pebbleMode optionally forces the Pebble merge strategy; the zero // value (Auto) lets the compactor choose. See WithPebbleCompactorMode. @@ -111,7 +137,7 @@ func (c *Compactor) inferEngineFromInputs() (c1zstore.Engine, error) { if entry == nil || entry.FilePath == "" { continue } - f, err := os.Open(entry.FilePath) // #nosec G304 - compaction inputs are caller-provided c1z paths. + f, err := os.Open(entry.FilePath) // #nosec G304,G703 -- compaction inputs are intentionally caller-provided c1z paths. if err != nil { return "", fmt.Errorf("infer compactor engine from %s: %w", entry.FilePath, err) } @@ -169,6 +195,22 @@ func WithTmpDir(tempDir string) Option { } } +// WithIncrementalExpansion enables diff-aware grant expansion during compaction. +// The compactor loads the graph from entries[0] via sync.GraphFromStore; +// missing, stale, incomplete, or inconsistent graphs safely fall back. The set of +// entitlements whose membership changed is derived from the applied increments +// during expansion (not supplied by the caller), so new members propagate. A +// new edge that closes a cycle falls back to full expansion; nil baseGraph +// (default) = full. +// +// Additions-only: a revocation-shaped change (a narrowed edge spec) auto-declines +// to full expansion; removals are not propagated incrementally. +func WithIncrementalExpansion() Option { + return func(c *Compactor) { + c.incrementalExpansion = true + } +} + // Deprecated: There is now only one compactor type, so this option is no longer needed. func WithCompactorType(compactorType CompactorType) Option { return func(c *Compactor) { @@ -205,6 +247,14 @@ func WithSkipGrantExpansion() Option { } } +// WithFailFastInvariants promotes every ingestion-invariant verdict to a hard +// failure on both incremental and full expansion paths. +func WithFailFastInvariants() Option { + return func(c *Compactor) { + c.failFastInvariants = true + } +} + func NewCompactor(ctx context.Context, outputDir string, compactableSyncs []*CompactableSync, opts ...Option) (*Compactor, func() error, error) { if len(compactableSyncs) < 2 { return nil, nil, ErrNotEnoughFilesToCompact @@ -255,10 +305,10 @@ func NewCompactor(ctx context.Context, outputDir string, compactableSyncs []*Com return c, cleanup, nil } -func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { +func (c *Compactor) Compact(ctx context.Context) (_ *CompactableSync, retErr error) { ctx, span := tracer.Start(ctx, "Compactor.Compact") var err error - defer func() { uotel.EndSpanWithError(span, err) }() + defer func() { uotel.EndSpanWithError(span, retErr) }() if len(c.entries) < 2 { return nil, nil } @@ -348,13 +398,12 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { opts = append(opts, dotc1z.WithDecoderPool(c.decoderPool)) } - if c.resolvedEngine() == c1zstore.EnginePebble { - // Force the resolved engine last so a stray engine passed via - // WithC1ZOptions cannot mislabel the artifact. - c.compactedC1z, err = dotc1z.NewStore(ctx, destFilePath, append(opts, dotc1z.WithEngine(c1zstore.EnginePebble))...) - } else { - c.compactedC1z, err = dotc1z.NewStore(ctx, destFilePath, opts...) - } + // Force the resolved engine last: a stray engine passed via + // WithC1ZOptions cannot mislabel the artifact, and the dotc1z + // engine default (Pebble) cannot leak into a SQLite compaction — + // the destination is a new file, so an engine-less open would + // otherwise create a v3 store under the SQLite merge path. + c.compactedC1z, err = dotc1z.NewStore(ctx, destFilePath, append(opts, dotc1z.WithEngine(c.resolvedEngine()))...) if err != nil { l.Error("doOneCompaction failed: could not create c1z file", zap.Error(err)) return nil, err @@ -363,9 +412,9 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { if c.compactedC1z == nil { return } - err := c.compactedC1z.Close(ctx) - if err != nil { - l.Error("compactor: error closing compacted c1z", zap.Error(err), zap.String("compacted_c1z_file", destFilePath)) + if closeErr := c.compactedC1z.Close(ctx); closeErr != nil { + l.Error("compactor: error closing compacted c1z", zap.Error(closeErr), zap.String("compacted_c1z_file", destFilePath)) + retErr = joinCompactorCloseError(retErr, closeErr, destFilePath) } }() var newSyncId string @@ -461,11 +510,21 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { c.compactedC1z = nil } + if c.incrementalExpansionRan { + if err := c.runIncrementalTestHook("before_publish"); err != nil { + return nil, err + } + } // Move last compacted file to the destination dir finalPath := path.Join(c.destDir, fmt.Sprintf("compacted-%s.c1z", newSyncId)) if err := cpFile(ctx, destFilePath, finalPath); err != nil { return nil, err } + if c.incrementalExpansionRan { + if err := c.runIncrementalTestHook("after_publish"); err != nil { + return nil, err + } + } if !filepath.IsAbs(finalPath) { abs, err := filepath.Abs(finalPath) @@ -477,8 +536,15 @@ func (c *Compactor) Compact(ctx context.Context) (*CompactableSync, error) { return &CompactableSync{FilePath: finalPath, SyncID: newSyncId}, nil } +func joinCompactorCloseError(retErr, closeErr error, artifactPath string) error { + if closeErr == nil { + return retErr + } + return errors.Join(retErr, fmt.Errorf("close compacted c1z %s: %w", artifactPath, closeErr)) +} + func cpFile(ctx context.Context, sourcePath string, destPath string) error { - err := os.Rename(sourcePath, destPath) + err := os.Rename(sourcePath, destPath) // #nosec G703 -- compaction source and destination paths are intentional API inputs. if err == nil { return nil } @@ -492,7 +558,7 @@ func cpFile(ctx context.Context, sourcePath string, destPath string) error { } defer source.Close() - destination, err := os.Create(destPath) + destination, err := os.Create(destPath) // #nosec G703 -- the caller intentionally selects the compacted artifact destination. if err != nil { return fmt.Errorf("failed to create destination file: %w", err) } @@ -565,8 +631,667 @@ func (c *Compactor) doOneCompaction(ctx context.Context, cs *CompactableSync) er return nil } +// expandGrantsIncremental runs a diff-aware expansion over the compacted c1z. +// errIncrementalFatal marks incremental-expansion errors that must FAIL the +// compaction rather than fall back to full expansion: the store is mid-teardown +// (or could not be restored to its ended state), so running the full path +// against it is unsafe. Every other error is safe to fall back on — the store +// was untouched or restored, and expanded-grant writes are idempotent. +var errIncrementalFatal = errors.New("incremental expansion: fatal") + +// errIncrementalDroppedEdgeDecline keeps the public revocation contract while +// giving observability a stable, more specific reason. +var errIncrementalDroppedEdgeDecline = fmt.Errorf("%w: dropped edge", expand.ErrIncrementalRevocationDecline) + +// Returns (true, nil) when it handled expansion. Errors come in three shapes: +// decline sentinels (ErrIncrementalFallback for a cycle, +// ErrIncrementalRevocationDecline for a narrowed edge) and plain errors both +// mean "fall back to full expansion" — the store is in the ended state the +// full path expects; errors wrapped in errIncrementalFatal mean the store's +// finalization failed and the compaction must fail. Finalization always runs +// on a detached context so a run-duration timeout can't abort it. +func (c *Compactor) expandGrantsIncremental(ctx context.Context, newSyncId string, compactionStart time.Time) (bool, error) { + // Classification only reads the caller-held graph. Defer the whale-sized + // clone until every cheap decline check passes; chronically ineligible + // inputs should not pay O(graph) memory and CPU before falling back. + base := c.incrementalBaseGraph + if err := base.ValidateCompleted(); err != nil { + return false, fmt.Errorf("incremental expansion: invalid base graph: %w", err) + } + if base.HasCollapsedCycles() { + return false, expand.ErrIncrementalFallback + } + + // Bound the walk by the remaining run duration; the walk polls ctx.Err(). + // Finalization uses detached contexts, so an expired walk deadline never + // aborts the end/cleanup/close. + walkCtx := ctx + if c.runDuration > 0 { + remaining := c.runDuration - time.Since(compactionStart) + if remaining <= 0 { + // Let the full path surface its canonical run-duration error. + return false, fmt.Errorf("incremental expansion: run duration expired before expansion") + } + var cancel context.CancelFunc + walkCtx, cancel = context.WithTimeout(ctx, remaining) + defer cancel() + } + + // The merge left the sync ended; resume it so grants can be written (end + + // close on the way out). Fetch the type first so resume finds the existing sync. + syncResp, err := c.compactedC1z.GetSync(walkCtx, reader_v2.SyncsReaderServiceGetSyncRequest_builder{SyncId: newSyncId}.Build()) + if err != nil { + return false, fmt.Errorf("incremental expansion: get sync: %w", err) + } + syncType := connectorstore.SyncType(syncResp.GetSync().GetSyncType()) + // ResumeSync, never StartOrResumeSync: the merge just produced this sync, so + // a failed lookup is an error, not a cue to start a fresh one. On Pebble, + // StartOrResumeSync's fallback runs ResetForNewSync, which would wipe + // everything the merge wrote. + if _, err := c.compactedC1z.ResumeSync(walkCtx, syncType, newSyncId); err != nil { + return false, fmt.Errorf("incremental expansion: resume sync: %w", err) + } + + // Every rule grant currently in the compacted c1z (base + merged + // increments) contributes to one or more edges. Multiple grants may describe + // different pieces of the SAME edge, so merge their specs before comparing + // them with the base graph. Comparing each piece independently turns an + // unchanged split filter (for example users + groups) into false narrowing. + currentEdges := make(map[[2]string]expand.NewEdge) + sourceEntitlements := make(map[string]*v2.Entitlement) + missingSourceEntitlements := make(map[string]struct{}) + for pe, err := range c.compactedC1z.Grants().PendingExpansion(walkCtx) { + if err != nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, fmt.Errorf("incremental expansion: enumerate pending: %w", err) + } + anno := pe.Annotation + if anno == nil { + continue + } + for _, src := range anno.GetEntitlementIds() { + if _, missing := missingSourceEntitlements[src]; missing { + continue + } + sourceEntitlement, cached := sourceEntitlements[src] + if !cached { + resp, getErr := c.compactedC1z.GetEntitlement(walkCtx, + reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ + EntitlementId: src, + }.Build()) + if status.Code(getErr) == codes.NotFound { + missingSourceEntitlements[src] = struct{}{} + ctxzap.Extract(ctx).Debug("incremental expansion: source entitlement not found, skipping edge", + zap.String("src_entitlement_id", src), + zap.String("dst_entitlement_id", pe.TargetEntitlementID)) + continue + } + if getErr != nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, fmt.Errorf("incremental expansion: get source entitlement %s: %w", src, getErr) + } + sourceEntitlement = resp.GetEntitlement() + sourceEntitlements[src] = sourceEntitlement + } + + sourceResourceID := sourceEntitlement.GetResource().GetId() + if sourceResourceID == nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, fmt.Errorf("incremental expansion: source entitlement resource id was nil") + } + if pe.PrincipalResourceTypeID != sourceResourceID.GetResourceType() || + pe.PrincipalResourceID != sourceResourceID.GetResource() { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, fmt.Errorf("incremental expansion: source entitlement resource id did not match grant principal id") + } + + curEdge := expand.NewEdge{ + SourceEntitlementID: src, + DestEntitlementID: pe.TargetEntitlementID, + Shallow: anno.GetShallow(), + ResourceTypeIDs: anno.GetResourceTypeIds(), + } + key := [2]string{src, pe.TargetEntitlementID} + if existing, ok := currentEdges[key]; ok { + currentEdges[key] = mergeCurrentEdgeSpecs(existing, curEdge) + } else { + curEdge.ResourceTypeIDs = append([]string(nil), curEdge.ResourceTypeIDs...) + currentEdges[key] = curEdge + } + } + } + + var newEdges []expand.NewEdge + currentBaseNodeEdges := make(map[[2]int]struct{}) + currentEdgeKeys := make([][2]string, 0, len(currentEdges)) + for key := range currentEdges { + currentEdgeKeys = append(currentEdgeKeys, key) + } + sort.Slice(currentEdgeKeys, func(i, j int) bool { + if currentEdgeKeys[i][0] != currentEdgeKeys[j][0] { + return currentEdgeKeys[i][0] < currentEdgeKeys[j][0] + } + return currentEdgeKeys[i][1] < currentEdgeKeys[j][1] + }) + for _, key := range currentEdgeKeys { + curEdge := currentEdges[key] + srcNode := base.GetNode(curEdge.SourceEntitlementID) + dstNode := base.GetNode(curEdge.DestEntitlementID) + if srcNode != nil && dstNode != nil && srcNode.Id != dstNode.Id { + currentBaseNodeEdges[[2]int{srcNode.Id, dstNode.Id}] = struct{}{} + } + baseEdge, inBase := baseGraphEdge(base, curEdge.SourceEntitlementID, curEdge.DestEntitlementID) + if !inBase { + newEdges = append(newEdges, curEdge) // brand-new edge + continue + } + // Existing edge: compare its combined current spec with the combined + // spec persisted in the base graph. + switch classifyEdgeSpecChange(baseEdge, curEdge) { + case edgeSpecNarrowed: + // Revocation-shaped (shallow-ified / filter tightened): can't + // remove grants incrementally — decline via the named hook (#6). + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, expand.ErrIncrementalRevocationDecline + case edgeSpecWidened: + // More members now qualify: re-expand (AddEdge folds the wider + // spec into the graph, deep-wins/unfiltered-wins). + newEdges = append(newEdges, curEdge) + case edgeSpecUnchanged: + // nothing to do + } + } + // PendingExpansion describes the complete current edge set. Check the + // reverse direction too: a base edge missing from current data is a + // revocation-shaped change and cannot be applied incrementally. + for _, edge := range base.Edges { + if _, ok := currentBaseNodeEdges[[2]int{edge.SourceID, edge.DestinationID}]; ok { + continue + } + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, errIncrementalDroppedEdgeDecline + } + + // Changed entitlements are derived from the applied increments (their + // grants' entitlement ids), not supplied by the caller — trust the data. + changedEntitlementIDs, err := c.changedEntitlementIDs(walkCtx) + if err != nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, err + } + + if len(newEdges) == 0 && len(changedEntitlementIDs) == 0 { + // Nothing changed relative to the base — its grants were already merged in. + base, err = base.Clone() + if err != nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, fmt.Errorf("incremental expansion: clone base graph: %w", err) + } + verification, err := c.runIncrementalInvariants(walkCtx, newSyncId, syncType) + if err != nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, err + } + return c.finishIncrementalExpansion(ctx, newSyncId, base, verification) + } + + base, err = base.Clone() + if err != nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, fmt.Errorf("incremental expansion: clone base graph: %w", err) + } + incrementalStore := sync.NewExpanderStore(c.compactedC1z) + if c.incrementalTestHook != nil { + incrementalStore = &incrementalFaultStore{ExpanderStore: incrementalStore, hook: c.incrementalTestHook} + } + ie := expand.NewIncrementalExpander(incrementalStore, base) + res, err := ie.ExpandChanges(walkCtx, newEdges, changedEntitlementIDs) + if err != nil { + // Restore the ended state so the full path re-runs against a consistent + // store — for the cycle decline and any real error alike. Writes are + // idempotent by grant identity, so partial progress is safe to re-cover. + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, err // sentinel or plain → caller falls back to full + } + if err := c.runIncrementalTestHook("after_walk"); err != nil { + return false, err + } + + ctxzap.Extract(ctx).Info("incremental grant expansion complete", + zap.Int("entitlements_walked", len(res.EntitlementsWalked)), + zap.Int("grants_written", res.GrantsWritten)) + verification, err := c.runIncrementalInvariants(walkCtx, newSyncId, syncType) + if err != nil { + if endErr := c.restoreEndedSync(ctx); endErr != nil { + return false, endErr + } + return false, err + } + return c.finishIncrementalExpansion(ctx, newSyncId, base, verification) +} + +type incrementalFaultStore struct { + expand.ExpanderStore + hook func(stage string) error + fired bool +} + +func (s *incrementalFaultStore) StoreExpandedGrants(ctx context.Context, grants ...*v2.Grant) error { + if err := s.ExpanderStore.StoreExpandedGrants(ctx, grants...); err != nil { + return err + } + if !s.fired { + s.fired = true + if err := s.hook("mid_expand_write"); err != nil { + return fmt.Errorf("%w: injected failure at mid_expand_write: %w", errIncrementalFatal, err) + } + } + return nil +} + +func (c *Compactor) runIncrementalTestHook(stage string) error { + if c.incrementalTestHook == nil { + return nil + } + if err := c.incrementalTestHook(stage); err != nil { + return fmt.Errorf("%w: injected failure at %s: %w", errIncrementalFatal, stage, err) + } + return nil +} + +func (c *Compactor) runIncrementalInvariants( + ctx context.Context, + syncID string, + syncType connectorstore.SyncType, +) (*c1zstore.IngestInvariantVerification, error) { + if writer, ok := c.compactedC1z.SyncMeta().(c1zstore.IngestInvariantVerificationWriter); ok { + if err := writer.ClearIngestInvariantVerification(ctx, syncID); err != nil { + return nil, fmt.Errorf("incremental expansion: clear invariant verification: %w", err) + } + } + verification, err := sync.RunIngestInvariantsWithVerification(ctx, c.compactedC1z, sync.IngestInvariantsPolicy{ + ActiveSyncID: syncID, + SyncType: syncType, + FailFast: c.failFastInvariants, + CompactionMerge: true, + }) + if err != nil { + return nil, fmt.Errorf("incremental expansion: ingest invariants: %w", err) + } + return verification, nil +} + +// persistGraphSidecar writes the post-expansion graph into the compacted c1z +// so the artifact carries its own base graph for the next incremental run. +// Best-effort: on failure the next run just falls back to full expansion. +func (c *Compactor) persistGraphSidecar(ctx context.Context, g *expand.EntitlementGraph, syncID string) { + gs, ok := c.compactedC1z.(sync.EntitlementGraphStore) + if !ok { + return + } + digestReader, ok := c.compactedC1z.(c1zstore.GrantGenerationDigestReader) + if !ok { + return + } + digest, found, err := digestReader.GrantGenerationDigest(ctx) + if err != nil || !found { + ctxzap.Extract(ctx).Warn("incremental expansion: sealed grant digest unavailable; graph will not be reusable", zap.Error(err)) + return + } + data, err := expand.MarshalGraphBlobWithGrantDigest(syncID, g, digest) + if err == nil { + err = gs.PutEntitlementGraphBlob(ctx, data) + } + if err != nil { + ctxzap.Extract(ctx).Warn("incremental expansion: persist graph sidecar failed", zap.Error(err)) + } +} + +// restoreEndedSync returns the compacted sync to the ended state the full path +// expects, after an incremental attempt that resumed it. Runs on a detached, +// timeout-bounded context so a cancelled parent can't strand the store +// mid-resume. Its failure is FATAL (errIncrementalFatal): the store is in an +// unknown state and the full path must not run against it. +func (c *Compactor) restoreEndedSync(ctx context.Context) error { + endCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dotc1z.FinalizeTimeout()) + defer cancel() + if err := c.compactedC1z.EndSync(endCtx); err != nil { + return fmt.Errorf("%w: restore ended sync: %w", errIncrementalFatal, err) + } + return nil +} + +// finishIncrementalExpansion ends, cleans up, and closes the store so the file +// is flushed before cpFile copies it — converging with the other compaction +// paths (Cleanup is a Pebble no-op today, kept for parity). Runs on a detached, +// timeout-bounded context so a cancelled or run-duration-expired parent can't +// abort finalization. All errors here are FATAL (errIncrementalFatal): the +// store is being torn down, so falling back to full expansion against it is +// not safe. +func (c *Compactor) finishIncrementalExpansion( + ctx context.Context, + syncID string, + graph *expand.EntitlementGraph, + verification *c1zstore.IngestInvariantVerification, +) (bool, error) { + finalizeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), dotc1z.FinalizeTimeout()) + defer cancel() + if err := c.compactedC1z.Cleanup(finalizeCtx); err != nil { + return false, fmt.Errorf("%w: cleanup: %w", errIncrementalFatal, err) + } + if err := c.runIncrementalTestHook("before_end_sync"); err != nil { + return false, err + } + if err := c.compactedC1z.EndSync(finalizeCtx); err != nil { + return false, fmt.Errorf("%w: end sync: %w", errIncrementalFatal, err) + } + if err := c.runIncrementalTestHook("after_end_sync"); err != nil { + return false, err + } + c.persistGraphSidecar(finalizeCtx, graph, syncID) + if err := c.runIncrementalTestHook("after_sidecar"); err != nil { + return false, err + } + if verification != nil { + if writer, ok := c.compactedC1z.SyncMeta().(c1zstore.IngestInvariantVerificationWriter); ok { + if err := writer.MarkIngestInvariantsVerified(finalizeCtx, syncID, *verification); err != nil { + ctxzap.Extract(ctx).Warn("incremental expansion: persist invariant verification failed; artifact remains unverified", zap.Error(err)) + } + } + } + if err := c.runIncrementalTestHook("after_marker"); err != nil { + return false, err + } + if err := c.runIncrementalTestHook("before_close"); err != nil { + return false, err + } + if err := c.compactedC1z.Close(finalizeCtx); err != nil { + return false, fmt.Errorf("%w: close: %w", errIncrementalFatal, err) + } + return true, nil +} + +// changedEntitlementIDs returns the entitlement ids whose grants changed in +// the applied increments, seeding the incremental walk. The fold collects +// them during its merge (no re-read, no-ops excluded); rebuild-mode +// compactions fall back to deriveChangedEntitlementIDs. +func (c *Compactor) changedEntitlementIDs(ctx context.Context) ([]string, error) { + if c.foldChangedEntitlementIDs != nil { + out := make([]string, 0, len(c.foldChangedEntitlementIDs)) + for id := range c.foldChangedEntitlementIDs { + out = append(out, id) + } + sort.Strings(out) + return out, nil + } + return c.deriveChangedEntitlementIDs(ctx) +} + +// deriveChangedEntitlementIDs is the no-fold fallback: re-open each increment +// (entries[1:]) and collect its grants' entitlement ids. +func (c *Compactor) deriveChangedEntitlementIDs(ctx context.Context) ([]string, error) { + if len(c.entries) < 2 { + return nil, nil + } + seen := make(map[string]struct{}) + for _, e := range c.entries[1:] { + // Same open options as doOneCompaction: honor the caller's tmp dir + // (extraction must not silently land in os.TempDir()) and parallel decode. + store, err := dotc1z.NewStore(ctx, e.FilePath, + dotc1z.WithTmpDir(c.tmpDir), + dotc1z.WithDecoderOptions(dotc1z.WithDecoderConcurrency(-1)), + dotc1z.WithReadOnly(true), + ) + if err != nil { + return nil, fmt.Errorf("incremental expansion: open increment %s: %w", e.SyncID, err) + } + err = collectGrantEntitlementIDs(ctx, store, e.SyncID, seen) + if closeErr := store.Close(ctx); closeErr != nil && err == nil { + err = fmt.Errorf("incremental expansion: close increment %s: %w", e.SyncID, closeErr) + } + if err != nil { + return nil, err + } + } + out := make([]string, 0, len(seen)) + for id := range seen { + out = append(out, id) + } + sort.Strings(out) + return out, nil +} + +// collectGrantEntitlementIDs adds every entitlement id that has a grant in the +// given sync to seen. +func collectGrantEntitlementIDs(ctx context.Context, store c1zstore.Store, syncID string, seen map[string]struct{}) error { + if err := store.SetCurrentSync(ctx, syncID); err != nil { + return fmt.Errorf("incremental expansion: set increment sync %s: %w", syncID, err) + } + pageToken := "" + for { + resp, err := store.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ + PageSize: 1000, + PageToken: pageToken, + }.Build()) + if err != nil { + return fmt.Errorf("incremental expansion: list increment grants for %s: %w", syncID, err) + } + for _, g := range resp.GetList() { + if id := g.GetEntitlement().GetId(); id != "" { + seen[id] = struct{}{} + } + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + return nil + } + } +} + +// baseGraphEdge returns the base graph's edge src->dst and whether one exists. +// Endpoints collapsed into one node (a fixed cycle) count as present with no +// distinct edge (nil), which classifyEdgeSpecChange treats as unchanged. +func baseGraphEdge(g *expand.EntitlementGraph, src, dst string) (*expand.Edge, bool) { + sn := g.GetNode(src) + dn := g.GetNode(dst) + if sn == nil || dn == nil { + return nil, false + } + if sn.Id == dn.Id { + return nil, true + } + dests, ok := g.SourcesToDestinations[sn.Id] + if !ok { + return nil, false + } + edgeID, ok := dests[dn.Id] + if !ok { + return nil, false + } + e, ok := g.Edges[edgeID] + if !ok { + return nil, false + } + return &e, true +} + +// mergeCurrentEdgeSpecs folds parallel connector rules for the same endpoints +// into the one effective graph edge AddEdge would build: deep wins over +// shallow, an unfiltered rule wins over filtered rules, and otherwise filters +// are unioned. +func mergeCurrentEdgeSpecs(left, right expand.NewEdge) expand.NewEdge { + out := left + out.Shallow = left.Shallow && right.Shallow + if len(left.ResourceTypeIDs) == 0 || len(right.ResourceTypeIDs) == 0 { + out.ResourceTypeIDs = nil + return out + } + + resourceTypeIDs := make(map[string]struct{}, len(left.ResourceTypeIDs)+len(right.ResourceTypeIDs)) + for _, id := range left.ResourceTypeIDs { + resourceTypeIDs[id] = struct{}{} + } + for _, id := range right.ResourceTypeIDs { + resourceTypeIDs[id] = struct{}{} + } + out.ResourceTypeIDs = make([]string, 0, len(resourceTypeIDs)) + for id := range resourceTypeIDs { + out.ResourceTypeIDs = append(out.ResourceTypeIDs, id) + } + sort.Strings(out.ResourceTypeIDs) + return out +} + +type edgeSpecChange int + +const ( + edgeSpecUnchanged edgeSpecChange = iota + edgeSpecWidened + edgeSpecNarrowed +) + +// classifyEdgeSpecChange compares an existing base edge's spec to the current +// (increment) spec. Narrowing (deep->shallow, filter tightened) is +// revocation-shaped; widening (shallow->deep, filter broadened) needs +// re-expansion. Any narrowing wins (safest: decline to full). +func classifyEdgeSpecChange(base *expand.Edge, cur expand.NewEdge) edgeSpecChange { + if base == nil { + return edgeSpecUnchanged // collapsed cycle: no distinct edge + } + widened, narrowed := false, false + if base.IsShallow && !cur.Shallow { + widened = true // shallow -> deep + } + if !base.IsShallow && cur.Shallow { + narrowed = true // deep -> shallow + } + rw, rn := compareResourceTypeFilter(base.ResourceTypeIDs, cur.ResourceTypeIDs) + widened = widened || rw + narrowed = narrowed || rn + switch { + case narrowed: + return edgeSpecNarrowed + case widened: + return edgeSpecWidened + default: + return edgeSpecUnchanged + } +} + +// compareResourceTypeFilter compares two principal-type filters where an empty +// filter means "all types" (the widest). Returns whether the current filter is +// wider and/or narrower than the base. +func compareResourceTypeFilter(base, cur []string) (bool, bool) { + var widened, narrowed bool + baseAll := len(base) == 0 + curAll := len(cur) == 0 + switch { + case baseAll && curAll: + return false, false + case baseAll && !curAll: + return false, true // all -> some + case !baseAll && curAll: + return true, false // some -> all + } + baseSet := make(map[string]struct{}, len(base)) + for _, t := range base { + baseSet[t] = struct{}{} + } + curSet := make(map[string]struct{}, len(cur)) + for _, t := range cur { + curSet[t] = struct{}{} + } + for t := range curSet { + if _, ok := baseSet[t]; !ok { + widened = true + } + } + for t := range baseSet { + if _, ok := curSet[t]; !ok { + narrowed = true + } + } + return widened, narrowed +} + func (c *Compactor) expandGrants(ctx context.Context, newSyncId string, compactionStart time.Time) error { l := ctxzap.Extract(ctx) + + // Diff-aware fast path: with a base graph, expand only what changed relative + // to it. Any doubt (cycle, error) falls through to full expansion below. + // Pebble-only: it reopens the ended compacted sync to write grants, which + // only Pebble supports; on other engines we degrade gracefully to full. + switch { + case !c.incrementalExpansion: + logIncrementalOutcome(ctx, "not_attempted", "not_requested") + case c.resolvedEngine() != c1zstore.EnginePebble: + logIncrementalOutcome(ctx, "not_attempted", "unsupported_engine", + zap.String("engine", string(c.resolvedEngine()))) + default: + baseGraph, loadErr := c.loadIncrementalBaseGraph(ctx) + if loadErr != nil { + logIncrementalOutcome(ctx, "fell_back", "base_graph_error", zap.Error(loadErr)) + break + } + if baseGraph == nil { + logIncrementalOutcome(ctx, "fell_back", "base_graph_missing_or_stale") + break + } + c.incrementalBaseGraph = baseGraph + done, err := c.expandGrantsIncremental(ctx, newSyncId, compactionStart) + switch { + case errors.Is(err, errIncrementalFatal): + // The store's finalization (or restore-to-ended) failed: it is in an + // unknown/torn-down state, so running full expansion against it is + // unsafe. Fail the compaction. + logIncrementalOutcome(ctx, "failed", "finalization_error", zap.Error(err)) + return fmt.Errorf("incremental grant expansion: %w", err) + case errors.Is(err, errIncrementalDroppedEdgeDecline): + logIncrementalOutcome(ctx, "declined", "dropped_edge") + case errors.Is(err, expand.ErrIncrementalRevocationDecline): + // Named revocation hook (#6): today declines to full; a future + // tombstone stage flips this one site to apply deletions. + logIncrementalOutcome(ctx, "declined", "revocation") + case errors.Is(err, expand.ErrIncrementalDenseChangeDecline): + logIncrementalOutcome(ctx, "declined", "dense_change") + case errors.Is(err, expand.ErrIncrementalFallback): + // New edge closed a cycle: full expansion handles cycles correctly. + logIncrementalOutcome(ctx, "declined", "cycle") + case err != nil: + // Pre-write or restored-state failure: the store is back in the + // ended state the full path expects, so falling back is safe. + logIncrementalOutcome(ctx, "fell_back", "incremental_error", zap.Error(err)) + case done: + // Incremental path already ended + closed the store; caller clears + // c.compactedC1z after return, same as the full path. + c.incrementalExpansionRan = true + logIncrementalOutcome(ctx, "succeeded", "none") + return nil + } + } + // Grant expansion doesn't use the connector interface at all, so giving syncer an empty connector is safe... for now. // If that ever changes, we should implement a file connector that is a wrapper around the reader. emptyConnector, err := sdk.NewEmptyConnector() @@ -588,6 +1313,23 @@ func (c *Compactor) expandGrants(ctx context.Context, newSyncId string, compacti // pinned by TestCompactionExpandToleratesMergeManufacturedExclusionConflicts. sync.WithCompactionMergedStore(), } + if c.failFastInvariants { + syncOpts = append(syncOpts, sync.WithFailFastInvariants()) + } + + // Keep the artifact's graph sidecar coherent with this full expansion: + // opted-in compactions preserve a fresh graph (so the incremental chain + // heals after a fallback); otherwise drop any sidecar inherited from a + // fold-copied base. Pebble-only: incremental expansion declines on other + // engines, and without a sidecar the preserved graph would only bloat + // the final sync token. + if c.incrementalExpansion && c.resolvedEngine() == c1zstore.EnginePebble { + syncOpts = append(syncOpts, sync.WithPreserveEntitlementGraph()) + } else if gs, ok := c.compactedC1z.(sync.EntitlementGraphStore); ok { + if err := gs.DeleteEntitlementGraphBlob(ctx); err != nil { + l.Warn("expandGrants: delete inherited graph sidecar failed", zap.Error(err)) + } + } compactionDuration := time.Since(compactionStart) runDuration := c.runDuration - compactionDuration @@ -614,9 +1356,74 @@ func (c *Compactor) expandGrants(ctx context.Context, newSyncId string, compacti l.Error("error syncing with grant expansion", zap.Error(err)) return err } + if c.resolvedEngine() == c1zstore.EnginePebble && c.pebbleMode != PebbleCompactorModeFold { + eng, ok := enginepkg.AsEngine(c.compactedC1z) + if !ok { + err := errors.New("invalidate expanded compaction source-cache state: compacted store is not a pebble engine") + return errors.Join(err, syncer.Close(ctx)) + } + if err := eng.InvalidateSourceCacheReplayState(ctx, true); err != nil { + err = fmt.Errorf("invalidate expanded compaction source-cache state: %w", err) + return errors.Join(err, syncer.Close(ctx)) + } + if !enginepkg.MarkStoreDirty(c.compactedC1z) { + err := errors.New("invalidate expanded compaction source-cache state: could not mark store dirty") + return errors.Join(err, syncer.Close(ctx)) + } + } if err := syncer.Close(ctx); err != nil { l.Error("error closing syncer", zap.Error(err)) return err } return nil } + +func logIncrementalOutcome(ctx context.Context, outcome, reason string, fields ...zap.Field) { + fields = append([]zap.Field{ + zap.String("incremental_expansion_outcome", outcome), + zap.String("incremental_expansion_reason", reason), + }, fields...) + ctxzap.Extract(ctx).Info("incremental grant expansion outcome", fields...) +} + +func (c *Compactor) loadIncrementalBaseGraph(ctx context.Context) (*expand.EntitlementGraph, error) { + if len(c.entries) == 0 || c.entries[0] == nil || c.entries[0].SyncID == "" { + return nil, fmt.Errorf("incremental expansion: compaction base is missing") + } + store, err := dotc1z.NewStore(ctx, c.entries[0].FilePath, + dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(c.tmpDir)) + if err != nil { + return nil, fmt.Errorf("incremental expansion: open base graph store: %w", err) + } + run, runErr := store.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + if runErr != nil { + _ = store.Close(ctx) + return nil, fmt.Errorf("incremental expansion: load base verification: %w", runErr) + } + // Both engines return (nil, nil) when the artifact holds no finished sync + // (e.g. an interrupted collection): decline to full expansion, don't panic. + if run == nil { + _ = store.Close(ctx) + return nil, fmt.Errorf("incremental expansion: base has no finished sync") + } + if run.ID != c.entries[0].SyncID || + !run.IsVerified() || + run.Generation != sync.IngestInvariantGeneration { + _ = store.Close(ctx) + return nil, fmt.Errorf("incremental expansion: base grant generation is not verified") + } + graph, graphErr := sync.GraphFromStore(ctx, store, c.entries[0].SyncID) + closeErr := store.Close(ctx) + if graphErr != nil { + return nil, fmt.Errorf("incremental expansion: load base graph: %w", graphErr) + } + if closeErr != nil { + return nil, fmt.Errorf("incremental expansion: close base graph store: %w", closeErr) + } + if graph != nil { + if err := graph.ValidateCompleted(); err != nil { + return nil, fmt.Errorf("incremental expansion: invalid base graph: %w", err) + } + } + return graph, nil +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go index c0ee8408..602f6df3 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/compactor_pebble.go @@ -27,10 +27,11 @@ import ( ) // WithEngine selects the storage engine for the compacted output. -// The default (unset) is sqlite, which is byte-identical to the -// historical compactor. EnginePebble produces a v3 Pebble c1z via a -// native record merge whose strategy (overlay / fold / kway) is -// resolved per run by resolvePebbleMode. +// The default (unset) follows the inputs — any Pebble input makes the +// output Pebble; all-SQLite inputs keep SQLite output, byte-identical +// to the historical compactor (see inferEngineFromInputs). EnginePebble +// produces a v3 Pebble c1z via a native record merge whose strategy +// (overlay / fold / kway) is resolved per run by resolvePebbleMode. // // This is the only supported way to choose the engine; an engine // passed through WithC1ZOptions does not select the compaction @@ -348,9 +349,9 @@ func ensurePebbleRegistered() error { } // compactableV3SyncType reports whether a v3 sync type is a compactable -// snapshot type. Diff syncs (partial_upserts / partial_deletions) are -// excluded — compaction folds full / resources-only / partial snapshots -// only, matching the sqlite source selection. +// snapshot type. Compaction folds full / resources-only / partial +// snapshots only (never unspecified/unknown types), matching the sqlite +// source selection. func compactableV3SyncType(t v3.SyncType) bool { switch t { case v3.SyncType_SYNC_TYPE_FULL, @@ -435,6 +436,17 @@ func selectSourceSyncFromManifest(path string) (manifestSourceSelection, bool) { return sel, true } +// joinSourceStoreCloseError keeps source-store Close errors as hard ownership +// failures even after the source has been fully consumed. Downgrading them +// would report success while hiding leaked Pebble resources or failed +// temporary-artifact cleanup. +func joinSourceStoreCloseError(retErr, closeErr error, sourcePath string) error { + if closeErr == nil { + return retErr + } + return errors.Join(retErr, fmt.Errorf("close source store %s: %w", sourcePath, closeErr)) +} + // compactPebbleFold is the in-place fold strategy (auto-selected for // large-base + small-partial inputs, or forced via // BATON_EXPERIMENTAL_PEBBLE_COMPACTOR=fold): the dest store is a copy @@ -444,8 +456,8 @@ func selectSourceSyncFromManifest(path string) (manifestSourceSelection, bool) { // - Base primary and index keys: zero writes — the data keyspace // carries no sync_id, so folding and the final rename touch none of // them. Work is O(partials), not O(base). -// - Partial winners are merged into the base keyspace via the -// engine's keep-newer path (Put*RecordsIfNewer), which compares +// - Partial winners are merged into the base keyspace via the raw +// keep-newer merge (mergeBucketRawIfNewer), which compares // discovered_at against the incumbent and maintains indexes with // point tombstones proportional to overridden records only. // - Tie semantics: a partial record with discovered_at EQUAL to the @@ -492,7 +504,7 @@ func (c *Compactor) compactPebbleFold(ctx context.Context) (string, error) { var convertedInputs []string defer func() { for _, path := range convertedInputs { - _ = os.Remove(path) + _ = os.Remove(path) // #nosec G703 -- paths come only from CreateTemp in the compactor temp directory. } }() for i := len(c.entries) - 1; i >= 1; i-- { @@ -527,18 +539,18 @@ func (c *Compactor) compactPebbleFold(ctx context.Context) (string, error) { } srcEng, ok := enginepkg.AsEngine(w) if !ok { - _ = w.Close(ctx) - return "", fmt.Errorf("compactPebbleFold: input %s is not a pebble c1z", sourcePath) + err := fmt.Errorf("compactPebbleFold: input %s is not a pebble c1z", sourcePath) + return "", joinSourceStoreCloseError(err, w.Close(ctx), sourcePath) } if srcSyncID == "" { rec, err := srcEng.LatestFinishedSyncRecord(ctx, compactableV3SyncType) if err != nil { - _ = w.Close(ctx) - return "", fmt.Errorf("compactPebbleFold: input %s: select compactable sync: %w", sourcePath, err) + err = fmt.Errorf("compactPebbleFold: input %s: select compactable sync: %w", sourcePath, err) + return "", joinSourceStoreCloseError(err, w.Close(ctx), sourcePath) } if rec == nil { - _ = w.Close(ctx) - return "", fmt.Errorf("compactPebbleFold: input %s has no finished compactable sync", sourcePath) + err := fmt.Errorf("compactPebbleFold: input %s has no finished compactable sync", sourcePath) + return "", joinSourceStoreCloseError(err, w.Close(ctx), sourcePath) } srcSyncID = rec.GetSyncId() unionType = unionV3SyncType(unionType, rec.GetType()) @@ -549,13 +561,50 @@ func (c *Compactor) compactPebbleFold(ctx context.Context) (string, error) { partialSyncIDs = append(partialSyncIDs, srcSyncID) partialTokens = append(partialTokens, readSourceSyncToken(ctx, srcEng, srcSyncID)) - mergeStats, mergeErr := mergepkg.MergeInto(ctx, destEng, []mergepkg.SourceSync{{Engine: srcEng, SyncID: srcSyncID}}, baseSyncID) + var mergeOpts []mergepkg.MergeOption + if c.incrementalExpansion { + mergeOpts = append(mergeOpts, mergepkg.WithGrantEntitlementIDs()) + } + mergeStats, mergeErr := mergepkg.MergeInto(ctx, destEng, []mergepkg.SourceSync{{Engine: srcEng, SyncID: srcSyncID}}, baseSyncID, mergeOpts...) foldStats.Add(mergeStats) - if cerr := w.Close(ctx); cerr != nil { - l.Error("compactPebbleFold: error closing source store", zap.Error(cerr), zap.String("file", sourcePath)) + closeErr := w.Close(ctx) + if closeErr != nil { + l.Error("compactPebbleFold: error closing source store", zap.Error(closeErr), zap.String("file", sourcePath)) } if mergeErr != nil { - return "", fmt.Errorf("compactPebbleFold: merge %s: %w", sourcePath, mergeErr) + mergeErr = fmt.Errorf("compactPebbleFold: merge %s: %w", sourcePath, mergeErr) + } + if err := joinSourceStoreCloseError(mergeErr, closeErr, sourcePath); err != nil { + return "", err + } + } + + // A fold inherits its base's validators, but the merged winners no longer + // represent that connector snapshot. Drop only the small manifest keyspace; + // retaining the existing source-scope indexes avoids an O(base) rewrite. + // + // DEPENDS ON THE OUTPUT BEING REPLAY-INELIGIBLE. The fold merge writes + // winner primaries through allBuckets(), which maintains by_parent and the + // grant index families but not by_source_scope. Keeping the base's + // source-scope entries therefore leaves them describing pre-fold state: + // an entry can point at a primary this fold just overwrote with a + // different (or absent) stamp, which is exactly the primary↔index + // biconditional that stageSourceScopeCleanup and validateReplaySourceScope + // assume. That is safe only because baseRec.SetCompacted(true) below marks + // the artifact compacted and validateReplaySourceEligible refuses compacted + // sources. If fold output is ever made eligible, this must become + // dropScopeIndexes=true (or the scope families must join the fold buckets) + // or replay will copy wrong rows silently. + if err := destEng.InvalidateSourceCacheReplayState(ctx, false); err != nil { + return "", fmt.Errorf("compactPebbleFold: invalidate source-cache replay state: %w", err) + } + + if c.incrementalExpansion { + // Hand the fold's changed-entitlement set to incremental expansion. + // Non-nil even when empty: nil means "no fold ran" (derive fallback). + c.foldChangedEntitlementIDs = foldStats.GrantEntitlementIDs + if c.foldChangedEntitlementIDs == nil { + c.foldChangedEntitlementIDs = map[string]struct{}{} } } @@ -677,9 +726,17 @@ func (c *Compactor) compactPebbleFold(ctx context.Context) (string, error) { // a lineage link would dangle, and the rebuild path's compacted // output carries no parent either. newSyncID := ksuid.New().String() + // The folded store is copied from the base, but the graph sidecar is + // stamped with that base sync ID and may no longer describe merged data. + // Drop it before publishing the fresh sync; a following expansion writes a + // new graph, while skip-expansion artifacts safely fall back next time. + if err := destEng.DeleteEntitlementGraphSidecar(ctx); err != nil { + return "", fmt.Errorf("compactPebbleFold: delete inherited entitlement graph: %w", err) + } baseRec.SetSyncId(newSyncID) baseRec.SetParentSyncId("") baseRec.SetType(unionType) + baseRec.SetCompacted(true) // The fold mutated the inherited base keyspace. Never publish the base // artifact's pre-fold verification as proof of the merged output; a later // expansion/invariant pass will write a fresh marker when one runs. @@ -881,7 +938,7 @@ func copyFileForFold(src, dst string) error { // readCompactionInputFormat reads the c1z header of path and returns its // on-disk format, rejecting anything that is not a supported v1/v3 c1z. func readCompactionInputFormat(path string) (dotc1z.C1ZFormat, error) { - f, err := os.Open(path) // #nosec G304 - compaction inputs are caller-provided c1z paths. + f, err := os.Open(path) // #nosec G304,G703 -- compaction inputs are intentionally caller-provided c1z paths. if err != nil { return dotc1z.C1ZFormatUnknown, fmt.Errorf("compactPebble: open input header %s: %w", path, err) } @@ -947,11 +1004,7 @@ func resolveSQLiteCompactionSyncID(ctx context.Context, store *dotc1z.C1File, ex } } if best == nil { - return "", fmt.Errorf( - "no finished compactable sync found in sqlite input (diff sync types %q/%q are not compactable)", - string(connectorstore.SyncTypePartialUpserts), - string(connectorstore.SyncTypePartialDeletions), - ) + return "", fmt.Errorf("no finished compactable sync found in sqlite input") } return best.GetId(), nil } @@ -1000,11 +1053,11 @@ func (c *Compactor) convertSQLiteInputToPebble(ctx context.Context, cs *Compacta } convertedPath := tmp.Name() if err := tmp.Close(); err != nil { - _ = os.Remove(convertedPath) + _ = os.Remove(convertedPath) // #nosec G703 -- convertedPath was returned by CreateTemp above. return "", fmt.Errorf("compactPebble: close conversion temp file: %w", err) } // ToPebble requires the destination path to not exist. - if err := os.Remove(convertedPath); err != nil { + if err := os.Remove(convertedPath); err != nil { // #nosec G703 -- convertedPath was returned by CreateTemp above. return "", fmt.Errorf("compactPebble: remove conversion temp placeholder: %w", err) } @@ -1020,16 +1073,16 @@ func (c *Compactor) convertSQLiteInputToPebble(ctx context.Context, cs *Compacta syncID, err := resolveSQLiteCompactionSyncID(ctx, sqliteStore, cs.SyncID) if err != nil { _ = store.Close(ctx) - _ = os.Remove(convertedPath) + _ = os.Remove(convertedPath) // #nosec G703 -- convertedPath was returned by CreateTemp above. return "", fmt.Errorf("compactPebble: select sqlite input sync %s: %w", cs.FilePath, err) } if _, err := sqliteStore.ToPebble(ctx, convertedPath, syncID, dotc1z.WithConvertTmpDir(c.tmpDir)); err != nil { _ = store.Close(ctx) - _ = os.Remove(convertedPath) + _ = os.Remove(convertedPath) // #nosec G703 -- convertedPath was returned by CreateTemp above. return "", fmt.Errorf("compactPebble: convert sqlite input %s to pebble: %w", cs.FilePath, err) } if err := store.Close(ctx); err != nil { - _ = os.Remove(convertedPath) + _ = os.Remove(convertedPath) // #nosec G703 -- convertedPath was returned by CreateTemp above. return "", fmt.Errorf("compactPebble: close sqlite input after conversion %s: %w", cs.FilePath, err) } return convertedPath, nil @@ -1037,8 +1090,8 @@ func (c *Compactor) convertSQLiteInputToPebble(ctx context.Context, cs *Compacta // compactPebble folds every input into the empty newSyncId on the // Pebble output via a native record merge: each input is opened, its -// latest finished compactable sync is selected (diff syncs excluded), -// and all are merged keeping the newest record per key. The output +// latest finished compactable sync is selected, and all are merged +// keeping the newest record per key. The output // sync_run's type and ended_at are then set to the union / max across // the inputs (mirroring the sqlite UpdateSync), and its stats are // recomputed. Inputs are merged in reverse entry order so the tie @@ -1088,7 +1141,7 @@ func (c *Compactor) compactPebble(ctx context.Context, newSyncId string) error { var convertedInputs []string defer func() { for _, path := range convertedInputs { - _ = os.Remove(path) + _ = os.Remove(path) // #nosec G703 -- paths come only from CreateTemp in the compactor temp directory. } }() for i := len(c.entries) - 1; i >= 0; i-- { @@ -1124,18 +1177,12 @@ func (c *Compactor) compactPebble(ctx context.Context, newSyncId string) error { continue } - source, syncType, endedAt, err := func() (mergepkg.SourceFile, v3.SyncType, time.Time, error) { + w, err := dotc1z.NewStore(ctx, sourcePath, dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(c.tmpDir), dotc1z.WithDecoderPool(c.decoderPool)) + if err != nil { + return fmt.Errorf("compactPebble: open input %s: %w", sourcePath, err) + } + source, syncType, endedAt, selectErr := func() (mergepkg.SourceFile, v3.SyncType, time.Time, error) { var zeroSource mergepkg.SourceFile - w, err := dotc1z.NewStore(ctx, sourcePath, dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(c.tmpDir), dotc1z.WithDecoderPool(c.decoderPool)) - if err != nil { - return zeroSource, v3.SyncType_SYNC_TYPE_UNSPECIFIED, time.Time{}, fmt.Errorf("compactPebble: open input %s: %w", sourcePath, err) - } - defer func() { - if cerr := w.Close(ctx); cerr != nil { - l.Error("compactPebble: error closing source store", zap.Error(cerr), zap.String("file", sourcePath)) - } - }() - srcEng, ok := enginepkg.AsEngine(w) if !ok { return zeroSource, v3.SyncType_SYNC_TYPE_UNSPECIFIED, time.Time{}, fmt.Errorf("compactPebble: input %s is not a pebble c1z", sourcePath) @@ -1145,7 +1192,7 @@ func (c *Compactor) compactPebble(ctx context.Context, newSyncId string) error { return zeroSource, v3.SyncType_SYNC_TYPE_UNSPECIFIED, time.Time{}, fmt.Errorf("compactPebble: select source sync for %s: %w", sourcePath, err) } if rec == nil { - return zeroSource, v3.SyncType_SYNC_TYPE_UNSPECIFIED, time.Time{}, fmt.Errorf("compactPebble: input %s has no finished compactable sync (diff syncs are not compactable)", sourcePath) + return zeroSource, v3.SyncType_SYNC_TYPE_UNSPECIFIED, time.Time{}, fmt.Errorf("compactPebble: input %s has no finished compactable sync", sourcePath) } // Record only (Path, SyncID, Stats) and fully close the store, @@ -1170,7 +1217,11 @@ func (c *Compactor) compactPebble(ctx context.Context, newSyncId string) error { } return source, rec.GetType(), endedAt, nil }() - if err != nil { + closeErr := w.Close(ctx) + if closeErr != nil { + l.Error("compactPebble: error closing source store", zap.Error(closeErr), zap.String("file", sourcePath)) + } + if err := joinSourceStoreCloseError(selectErr, closeErr, sourcePath); err != nil { return err } @@ -1227,6 +1278,13 @@ func (c *Compactor) compactPebble(ctx context.Context, newSyncId string) error { return fmt.Errorf("compactPebble: merge: %w", err) } + // K-way and overlay materialize into a fresh store. They must not publish + // either inherited validators or source-scope indexes for their merged + // winners; range tombstones keep this independent of output row count. + if err := destEng.InvalidateSourceCacheReplayState(ctx, true); err != nil { + return fmt.Errorf("compactPebble: invalidate source-cache replay state: %w", err) + } + if err := rebuildCompactedGrantDigests(ctx, destEng); err != nil { return fmt.Errorf("compactPebble: %w", err) } @@ -1239,6 +1297,7 @@ func (c *Compactor) compactPebble(ctx context.Context, newSyncId string) error { return fmt.Errorf("compactPebble: load dest sync_run: %w", err) } rec.SetType(unionType) + rec.SetCompacted(true) if !maxEnded.IsZero() { rec.SetEndedAt(timestamppb.New(maxEnded)) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go index 33d9be54..98722f6e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/bucket_plans.go @@ -14,10 +14,18 @@ type bucketPlan struct { upper []byte } -// buildBucketPlans returns the set of (lower, upper) excise spans that -// together cover every key in the engine. A v3 Pebble c1z holds one -// sync and keys carry no sync_id, so this is the whole keyspace. The -// order is fixed and deterministic so logs and tests are stable. +// buildBucketPlans returns the (lower, upper) excise spans this compaction +// carries forward. A v3 Pebble c1z holds one sync and keys carry no sync_id, +// so each span covers a whole family. The order is fixed and deterministic so +// logs and tests are stable. +// +// This is deliberately not every family in the engine: the source-cache +// manifest and the three by_source_scope index families are absent, so they +// are dropped rather than copied. That is the intended outcome — a compacted +// artifact is not a replay source (validateReplaySourceEligible), and the +// callers pair this with InvalidateSourceCacheReplayState(ctx, true). A new +// family added to the engine does need a span here, or its keys will be lost +// on compaction without any error. func buildBucketPlans() []bucketPlan { return []bucketPlan{ { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/doc.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/doc.go index 36c98a2b..fc0e4f8e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/doc.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/doc.go @@ -55,9 +55,9 @@ // // Not a rebuild. The dest store starts as a byte copy of the base // input, the output adopts the base sync's id, and each partial's -// records are streamed into the base keyspace through the engine's -// keep-newer puts (Put*RecordsIfNewer), which resolve conflicts -// against incumbents by discovered_at and maintain indexes with point +// records are streamed into the base keyspace through the raw +// keep-newer merge (mergeBucketRawIfNewer), which resolves conflicts +// against incumbents by discovered_at and maintains indexes with point // tombstones for overridden records only. Base records are never // read, decoded, or rewritten, and the envelope save splices the // base's unchanged zstd frames instead of re-encoding them — total diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/fold_commit.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/fold_commit.go new file mode 100644 index 00000000..e0375007 --- /dev/null +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/fold_commit.go @@ -0,0 +1,21 @@ +package pebble + +import ( + cpebble "github.com/cockroachdb/pebble/v2" + + enginepkg "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" +) + +// foldCommitFailure is an explicit internal injection argument, not mutable +// engine state. Production callers pass nil; focused tests pass a deterministic +// failure into the same ownership/commit code. +type foldCommitFailure func() error + +func commitFoldBatch(batch *enginepkg.FoldBatch, opts *cpebble.WriteOptions, before foldCommitFailure) error { + if before != nil { + if err := before(); err != nil { + return err + } + } + return batch.Commit(opts) +} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/kway.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/kway.go index cea0241e..46b66e2c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/kway.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/kway.go @@ -228,11 +228,13 @@ type sourceChunk struct { dir string // chunk-scoped unpack dir; empty when no Path sources } -func (c *sourceChunk) close() { - closeSourceHandles(c.handles) +func (c *sourceChunk) close() error { + err := closeSourceHandles(c.handles) + c.handles = nil if c.dir != "" { _ = os.RemoveAll(c.dir) } + return err } // closeAsync closes the chunk's engines synchronously (file locks and @@ -240,9 +242,11 @@ func (c *sourceChunk) close() { // directory to rm, so the unlink storm of an unpacked Pebble tree // happens off the merge's critical path. Benchmarks on APFS showed // in-loop chunk deletion costing a measurable slice of merge time. -func (c *sourceChunk) closeAsync(rm *asyncRemover) { - closeSourceHandles(c.handles) +func (c *sourceChunk) closeAsync(rm *asyncRemover) error { + err := closeSourceHandles(c.handles) + c.handles = nil rm.remove(c.dir) + return err } // asyncRemover deletes directory trees on background goroutines. @@ -270,8 +274,7 @@ func (d *asyncRemover) wait() { d.wg.Wait() } func openSourceChunk(ctx context.Context, tmpDir string, sources []SourceFile, baseRank int) (*sourceChunk, error) { chunk := &sourceChunk{handles: make([]sourceHandle, 0, len(sources))} cleanupOnError := func(err error) error { - chunk.close() - return err + return errors.Join(err, chunk.close()) } for i, source := range sources { if source.Engine != nil { @@ -311,8 +314,11 @@ func openSourceChunk(ctx context.Context, tmpDir string, sources []SourceFile, b } eng, ok := enginepkg.AsEngine(w) if !ok { - _ = w.Close(ctx) - return nil, cleanupOnError(fmt.Errorf("kway merge: input is not pebble: %s", source.Path)) + inputErr := fmt.Errorf("kway merge: input is not pebble: %s", source.Path) + if closeErr := w.Close(ctx); closeErr != nil { + inputErr = errors.Join(inputErr, fmt.Errorf("kway merge: close input %s: %w", source.Path, closeErr)) + } + return nil, cleanupOnError(inputErr) } store := w chunk.handles = append(chunk.handles, sourceHandle{ @@ -328,12 +334,27 @@ func openSourceChunk(ctx context.Context, tmpDir string, sources []SourceFile, b return chunk, nil } -func closeSourceHandles(handles []sourceHandle) { +func closeSourceHandles(handles []sourceHandle) error { + var retErr error for _, handle := range handles { if handle.close != nil { - _ = handle.close() + retErr = errors.Join(retErr, handle.close()) } } + return retErr +} + +func finishChunkRunFile(run runFile, buildErr, closeErr error) (runFile, error) { + retErr := errors.Join(buildErr, closeErr) + if retErr == nil { + return run, nil + } + if buildErr == nil && run.path != "" { + if removeErr := os.Remove(run.path); removeErr != nil && !errors.Is(removeErr, os.ErrNotExist) { + retErr = errors.Join(retErr, fmt.Errorf("remove unpublished run file %s: %w", run.path, removeErr)) + } + } + return runFile{}, retErr } func buildChunkRunFileFromSources( @@ -349,8 +370,8 @@ func buildChunkRunFileFromSources( if err != nil { return runFile{}, err } - defer chunk.closeAsync(rm) - return buildChunkRunFileFromHandles(ctx, tmpDir, chunk.handles, name, buckets) + run, buildErr := buildChunkRunFileFromHandles(ctx, tmpDir, chunk.handles, name, buckets) + return finishChunkRunFile(run, buildErr, chunk.closeAsync(rm)) } func buildChunkRunFileFromHandles( @@ -403,13 +424,12 @@ func mergeSourceChunkToPebble( if err != nil { return err } - defer chunk.closeAsync(rm) for _, bucket := range buckets { if err := materializeSourceBucketToPebble(ctx, dest, tmpDir, chunk.handles, bucket, stats); err != nil { - return err + return errors.Join(err, chunk.closeAsync(rm)) } } - return nil + return chunk.closeAsync(rm) } type countingWriter struct { diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/merge.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/merge.go index 627ee905..d48eaf73 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/merge.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/merge.go @@ -18,6 +18,23 @@ type SourceSync struct { SyncID string } +type mergeOptions struct { + collectGrantEntitlementIDs bool +} + +// MergeOption enables optional MergeInto behavior without breaking callers +// that use the original four-argument API. +type MergeOption func(*mergeOptions) + +// WithGrantEntitlementIDs records the entitlement IDs of grant records that +// MergeInto actually writes. The incremental expander uses these IDs as its +// changed-node seeds. +func WithGrantEntitlementIDs() MergeOption { + return func(opts *mergeOptions) { + opts.collectGrantEntitlementIDs = true + } +} + // FoldStats reports what a MergeInto call overrode in the destination // keyspace. DeadBytes is the exact raw size (keys + values) of the // incumbent records — and their derived index keys — that the fold @@ -56,6 +73,9 @@ type FoldStats struct { // (Engine.InvalidateGrantDigestPartitions + // Engine.RepairMissingGrantDigests), instead of the whole file. TouchedGrantPartitions map[string]struct{} + // GrantEntitlementIDs: distinct entitlement ids of applied grant records + // (no-ops excluded). Seeds incremental expansion without re-reading inputs. + GrantEntitlementIDs map[string]struct{} } func (s *FoldStats) Add(o FoldStats) { @@ -74,6 +94,24 @@ func (s *FoldStats) Add(o FoldStats) { } s.TouchedGrantPartitions[p] = struct{}{} } + for id := range o.GrantEntitlementIDs { + s.noteGrantEntitlementID([]byte(id)) + } +} + +// noteGrantEntitlementID records one applied grant's entitlement id; +// read-before-insert keeps repeats allocation-free. +func (s *FoldStats) noteGrantEntitlementID(id []byte) { + if len(id) == 0 { + return + } + if _, ok := s.GrantEntitlementIDs[string(id)]; ok { + return + } + if s.GrantEntitlementIDs == nil { + s.GrantEntitlementIDs = make(map[string]struct{}) + } + s.GrantEntitlementIDs[string(id)] = struct{}{} } func (s *FoldStats) bumpAdded(bucket string, n int64) { @@ -136,8 +174,14 @@ func (s *FoldStats) bumpReplaced(bucket string, n int64) { // (Engine.BuildGrantDigests rebuilds both keyspaces atomically from // scratch, so no separate drop is needed even then — see // compactPebbleFold). -func MergeInto(ctx context.Context, dest *enginepkg.Engine, sources []SourceSync, destSyncID string) (FoldStats, error) { +func MergeInto(ctx context.Context, dest *enginepkg.Engine, sources []SourceSync, destSyncID string, options ...MergeOption) (FoldStats, error) { var stats FoldStats + opts := mergeOptions{} + for _, option := range options { + if option != nil { + option(&opts) + } + } if dest == nil { return stats, errors.New("synccompactor/pebble.MergeInto: dest engine is nil") } @@ -159,7 +203,7 @@ func MergeInto(ctx context.Context, dest *enginepkg.Engine, sources []SourceSync if s.Engine == nil || s.SyncID == "" { continue } - srcStats, err := mergeOneSource(ctx, dest, s, destSyncID) + srcStats, err := mergeOneSource(ctx, dest, s, destSyncID, opts.collectGrantEntitlementIDs) stats.Add(srcStats) if err != nil { return stats, fmt.Errorf("merge source %s: %w", s.SyncID, err) @@ -187,13 +231,13 @@ const mergeRawFlushRecords = 32768 // newer wins, replacing the value and swapping the incumbent's // derived index keys for the new value's (point deletes // proportional to overridden records only). Ties keep the -// incumbent, mirroring the engine's Put*RecordsIfNewer rule — -// missing discovered_at scans as 0, reproducing its nil-timestamp -// ordering ("never overwrite an incumbent, always fill a hole"). -func mergeOneSource(ctx context.Context, dest *enginepkg.Engine, s SourceSync, destSyncID string) (FoldStats, error) { +// incumbent — missing discovered_at scans as 0, giving +// nil-timestamp ordering ("never overwrite an incumbent, always +// fill a hole"). +func mergeOneSource(ctx context.Context, dest *enginepkg.Engine, s SourceSync, destSyncID string, collectGrantEntitlementIDs bool) (FoldStats, error) { var stats FoldStats for _, bucket := range allBuckets() { - bucketStats, err := mergeBucketRawIfNewer(ctx, dest, s.Engine, bucket) + bucketStats, err := mergeBucketRawIfNewer(ctx, dest, s.Engine, bucket, collectGrantEntitlementIDs) stats.Add(bucketStats) if err != nil { return stats, fmt.Errorf("merge %s: %w", bucket.name, err) @@ -202,7 +246,18 @@ func mergeOneSource(ctx context.Context, dest *enginepkg.Engine, s SourceSync, d return stats, nil } -func mergeBucketRawIfNewer(ctx context.Context, dest *enginepkg.Engine, src *enginepkg.Engine, bucket bucketSpec) (FoldStats, error) { +func mergeBucketRawIfNewer(ctx context.Context, dest *enginepkg.Engine, src *enginepkg.Engine, bucket bucketSpec, collectGrantEntitlementIDs bool) (FoldStats, error) { + return mergeBucketRawIfNewerWithCommitFailure(ctx, dest, src, bucket, collectGrantEntitlementIDs, nil) +} + +func mergeBucketRawIfNewerWithCommitFailure( + ctx context.Context, + dest *enginepkg.Engine, + src *enginepkg.Engine, + bucket bucketSpec, + collectGrantEntitlementIDs bool, + beforeCommit foldCommitFailure, +) (FoldStats, error) { var stats FoldStats lower, upper := bucket.syncRange() iter, err := src.NewIter(&pebble.IterOptions{LowerBound: lower, UpperBound: upper}) @@ -230,7 +285,7 @@ func mergeBucketRawIfNewer(ctx context.Context, dest *enginepkg.Engine, src *eng } // NoSync: the fold's envelope save checkpoints (which flushes // and fsyncs) before anything depends on these writes. - if err := batch.Commit(pebble.NoSync); err != nil { + if err := commitFoldBatch(batch, pebble.NoSync, beforeCommit); err != nil { return err } _ = batch.Close() @@ -287,6 +342,9 @@ func mergeBucketRawIfNewer(ctx context.Context, dest *enginepkg.Engine, src *eng if err := batch.Set(key, value); err != nil { return stats, err } + // Applied grant (skips continued above): count it toward the + // digest-repair signal and collect its entitlement id for + // incremental expansion — both during the read the fold already does. if bucket.id == runBucketGrants { stats.GrantWrites++ if partition, ok := enginepkg.GrantPartitionFromPrimaryKey(key); ok { @@ -295,6 +353,13 @@ func mergeBucketRawIfNewer(ctx context.Context, dest *enginepkg.Engine, src *eng } stats.TouchedGrantPartitions[partition] = struct{}{} } + if collectGrantEntitlementIDs { + _, _, entID, _, _, _, scanErr := scanGrantIndexFieldsBytes(value) + if scanErr != nil { + return stats, scanErr + } + stats.noteGrantEntitlementID(entID) + } } if err := forEachIndexKeyFromRaw(bucket, key, lower, value, &scratch, nil, setIndexKey); err != nil { return stats, err diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/overlay.go b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/overlay.go index 7bbcda94..b0aae000 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/overlay.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/synccompactor/pebble/overlay.go @@ -351,6 +351,17 @@ func bucketIndexCopySpecs(bucket bucketSpec) []indexFamilyCopySpec { // whole-source SST path ingested. Stats for the bucket are zeroed; the // blind kway materialization recounts from scratch. func overlayRestartBucket(ctx context.Context, dest *enginepkg.Engine, bucket bucketSpec, writer *overlayBucketRawWriter, stats *mergeStatsAccumulator) error { + return overlayRestartBucketWithCommitFailure(ctx, dest, bucket, writer, stats, nil) +} + +func overlayRestartBucketWithCommitFailure( + ctx context.Context, + dest *enginepkg.Engine, + bucket bucketSpec, + writer *overlayBucketRawWriter, + stats *mergeStatsAccumulator, + beforeCommit foldCommitFailure, +) error { if err := ctx.Err(); err != nil { return err } @@ -366,7 +377,7 @@ func overlayRestartBucket(ctx context.Context, dest *enginepkg.Engine, bucket bu return err } } - if err := b.Commit(cpebble.NoSync); err != nil { + if err := commitFoldBatch(b, cpebble.NoSync, beforeCommit); err != nil { return err } stats.resetBucket(bucket.id) @@ -466,8 +477,7 @@ func MergeFilesIntoOverlay(ctx context.Context, dest *enginepkg.Engine, sources if err != nil { return nil, err } - if err := func() error { - defer chunk.closeAsync(rm) + workErr := func() error { for _, source := range chunk.handles { for bucketIdx, bucket := range overlayBuckets { st := &states[bucketIdx] @@ -620,7 +630,8 @@ func MergeFilesIntoOverlay(ctx context.Context, dest *enginepkg.Engine, sources kwayRunFiles = append(kwayRunFiles, run) } return nil - }(); err != nil { + }() + if err := errors.Join(workErr, chunk.closeAsync(rm)); err != nil { return nil, err } chunkIdx++ @@ -720,16 +731,14 @@ func overlayBackfillRestartedChunks( if err != nil { return err } - run, err := func() (runFile, error) { - defer chunk.closeAsync(rm) - return buildChunkRunFileFromHandles( - ctx, - tmpDir, - chunk.handles, - fmt.Sprintf("overlay-backfill-%04d", len(*kwayRunFiles)), - needed, - ) - }() + run, buildErr := buildChunkRunFileFromHandles( + ctx, + tmpDir, + chunk.handles, + fmt.Sprintf("overlay-backfill-%04d", len(*kwayRunFiles)), + needed, + ) + run, err = finishChunkRunFile(run, buildErr, chunk.closeAsync(rm)) if err != nil { return err } @@ -1751,6 +1760,10 @@ func (w *overlayBucketRawWriter) replaceRaw(ctx context.Context, bucket bucketSp } func (w *overlayBucketRawWriter) flush(ctx context.Context) error { + return w.flushWithCommitFailure(ctx, nil) +} + +func (w *overlayBucketRawWriter) flushWithCommitFailure(ctx context.Context, beforeCommit foldCommitFailure) error { if w == nil || w.count == 0 { return nil } @@ -1758,12 +1771,17 @@ func (w *overlayBucketRawWriter) flush(ctx context.Context) error { return err } opts := cpebble.NoSync - if err := w.primary.Commit(opts); err != nil { + if err := commitFoldBatch(w.primary, opts, beforeCommit); err != nil { return err } - if err := w.index.Commit(opts); err != nil { + if err := commitFoldBatch(w.index, opts, beforeCommit); err != nil { return err } + // Release the committed batches before minting replacements: Commit + // does not return the batch to pebble's pool, Close does. (On the + // error returns above, cleanup() closes whatever is still held.) + _ = w.primary.Close() + _ = w.index.Close() w.primary = w.dest.NewFoldBatch() w.index = w.dest.NewFoldBatch() w.count = 0 diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/differ.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/differ.go deleted file mode 100644 index c1d76e93..00000000 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/local/differ.go +++ /dev/null @@ -1,84 +0,0 @@ -package local - -import ( - "context" - "errors" - "sync" - "time" - - v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" - "github.com/conductorone/baton-sdk/pkg/dotc1z" - "github.com/conductorone/baton-sdk/pkg/tasks" - "github.com/conductorone/baton-sdk/pkg/types" - "github.com/conductorone/baton-sdk/pkg/uotel" - "github.com/conductorone/baton-sdk/pkg/uotel/uotelzap" - "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" - "go.opentelemetry.io/otel/trace" - "go.uber.org/zap" -) - -type localDiffer struct { - dbPath string - o sync.Once - - baseSyncID string - appliedSyncID string -} - -func (m *localDiffer) GetTempDir() string { - return "" -} - -func (m *localDiffer) ShouldDebug() bool { - return false -} - -func (m *localDiffer) Next(ctx context.Context) (*v1.Task, time.Duration, error) { - var task *v1.Task - m.o.Do(func() { - task = v1.Task_builder{ - CreateSyncDiff: &v1.Task_CreateSyncDiffTask{}, - }.Build() - }) - return task, 0, nil -} - -func (m *localDiffer) Process(ctx context.Context, task *v1.Task, cc types.ConnectorClient) error { - ctx, span := tracer.Start(ctx, "localDiffer.Process", trace.WithNewRoot()) - ctx = uotelzap.WithSpanLogFields(ctx) - var err error - defer func() { uotel.EndSpanWithError(span, err) }() - log := ctxzap.Extract(ctx) - - if m.baseSyncID == "" || m.appliedSyncID == "" { - return errors.New("missing base sync ID or applied sync ID") - } - - file, err := dotc1z.NewStore(ctx, m.dbPath) - if err != nil { - return err - } - - newSyncID, err := file.FileOps().GenerateSyncDiff(ctx, m.baseSyncID, m.appliedSyncID) - if err != nil { - return err - } - - if err := file.Close(ctx); err != nil { - log.Error("failed to close store", zap.Error(err)) - return err - } - - log.Info("generated diff of syncs", zap.String("new_sync_id", newSyncID)) - - return nil -} - -// NewDiffer returns a task manager that queues a revoke task. -func NewDiffer(ctx context.Context, dbPath string, baseSyncID string, appliedSyncID string) tasks.Manager { - return &localDiffer{ - dbPath: dbPath, - baseSyncID: baseSyncID, - appliedSyncID: appliedSyncID, - } -} diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/tasks.go b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/tasks.go index 73294fa0..ac7d6565 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/tasks/tasks.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/tasks/tasks.go @@ -68,8 +68,6 @@ func Is(task *v1.Task, target taskTypes.TaskType) bool { return actualType == v1.Task_ActionInvoke_case case taskTypes.ActionStatusType: return actualType == v1.Task_ActionStatus_case - case taskTypes.CreateSyncDiff: - return actualType == v1.Task_CreateSyncDiff_case case taskTypes.ListEventFeedsType: return actualType == v1.Task_ListEventFeeds_case case taskTypes.ListEventsType: @@ -125,8 +123,6 @@ func GetType(task *v1.Task) taskTypes.TaskType { return taskTypes.ActionInvokeType case v1.Task_ActionStatus_case: return taskTypes.ActionStatusType - case v1.Task_CreateSyncDiff_case: - return taskTypes.CreateSyncDiff case v1.Task_ListEventFeeds_case: return taskTypes.ListEventFeedsType case v1.Task_ListEvents_case: diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/types/tasks/tasks.go b/vendor/github.com/conductorone/baton-sdk/pkg/types/tasks/tasks.go index e35d82cf..6a54c04c 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/types/tasks/tasks.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/types/tasks/tasks.go @@ -66,8 +66,6 @@ func (tt TaskType) String() string { return "invoke_action" case ActionStatusType: return "action_status" - case CreateSyncDiff: - return "create_sync_diff" default: return "unknown" } @@ -104,7 +102,7 @@ const ( ActionGetSchemaType ActionInvokeType ActionStatusType - CreateSyncDiff + _ // was CreateSyncDiff; placeholder pins the ordinals below to their released values ListStaticEntitlementsType IssueCredentialType ) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/client.go b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/client.go index df9474ef..3237ebcf 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/client.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/client.go @@ -104,17 +104,53 @@ func NewClient(ctx context.Context, options ...Option) (*http.Client, error) { } type icache interface { - Get(req *http.Request) (*http.Response, error) - Set(req *http.Request, value *http.Response) error + Get(req *http.Request, opts ...CacheOption) (*http.Response, error) + Set(req *http.Request, value *http.Response, opts ...CacheOption) error Clear(ctx context.Context) error Stats(ctx context.Context) CacheStats } +type cacheKeyConfig struct { + headers []string +} + +// CacheOption configures how CreateCacheKey computes its key, beyond the +// default set of headers (Accept, Content-Type, Cookie, Range). Kept as an +// interface so future dimensions (TTL, query-param keying, etc.) can be +// added without changing CreateCacheKey's or icache's signatures again. +type CacheOption interface { + applyCache(*cacheKeyConfig) +} + +type cacheKeyHeadersOption []string + +func (o cacheKeyHeadersOption) applyCache(c *cacheKeyConfig) { + c.headers = append(c.headers, o...) +} + +// CacheKeyHeaders returns a CacheOption that folds the named headers into +// the cache key computed by CreateCacheKey (and by GoCache/DBCache's +// Get/Set), beyond the default set (Accept, Content-Type, Cookie, Range). +// The value folded in is always read from req.Header at key-computation +// time, so the key can never describe a value other than the one actually +// present on the request. Named headers must therefore be set on the +// request before it reaches the cache lookup; a header only added by a +// transport-level RoundTripper or a cookie jar after that point is not +// seen. +func CacheKeyHeaders(headers ...string) CacheOption { + return cacheKeyHeadersOption(headers) +} + // CreateCacheKey generates a cache key based on the request URL, query parameters, and headers. -func CreateCacheKey(req *http.Request) (string, error) { +func CreateCacheKey(req *http.Request, opts ...CacheOption) (string, error) { if req == nil { return "", fmt.Errorf("request is nil") } + var cfg cacheKeyConfig + for _, o := range opts { + o.applyCache(&cfg) + } + var sortedParams []string // Normalize the URL path path := strings.ToLower(req.URL.Path) @@ -130,13 +166,33 @@ func CreateCacheKey(req *http.Request) (string, error) { queryString := strings.Join(sortedParams, "&") // Include relevant headers in the cache key var headerParts []string + seenHeaders := map[string]bool{ + "Accept": true, + "Content-Type": true, + "Cookie": true, + "Range": true, + } for key, values := range req.Header { for _, value := range values { - if key == "Accept" || key == "Content-Type" || key == "Cookie" || key == "Range" { + if seenHeaders[key] { headerParts = append(headerParts, fmt.Sprintf("%s=%s", key, value)) } } } + // Opted-in headers are folded in on top of the default set above. + // seenHeaders already marks the default set, and gets marked as each + // opted-in header is processed, so a header named in cfg.headers -- by + // one CacheOption or by several -- is never folded in more than once. + for _, h := range cfg.headers { + key := http.CanonicalHeaderKey(h) + if seenHeaders[key] { + continue + } + seenHeaders[key] = true + for _, value := range req.Header[key] { + headerParts = append(headerParts, fmt.Sprintf("%s=%s", key, value)) + } + } sort.Strings(headerParts) headersString := strings.Join(headerParts, "&") diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/dbcache.go b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/dbcache.go index 1eeba061..860af5cf 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/dbcache.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/dbcache.go @@ -190,12 +190,12 @@ func (d *DBCache) removeDB(ctx context.Context) error { } // Get returns cached response (if exists). -func (d *DBCache) Get(req *http.Request) (*http.Response, error) { +func (d *DBCache) Get(req *http.Request, opts ...CacheOption) (*http.Response, error) { var ( isFound = false resp *http.Response ) - key, err := CreateCacheKey(req) + key, err := CreateCacheKey(req, opts...) if err != nil { return nil, err } @@ -250,8 +250,8 @@ func (d *DBCache) pick(ctx context.Context, key string) ([]byte, error) { } // Set stores and save response in the db. -func (d *DBCache) Set(req *http.Request, value *http.Response) error { - key, err := CreateCacheKey(req) +func (d *DBCache) Set(req *http.Request, value *http.Response, opts ...CacheOption) error { + key, err := CreateCacheKey(req, opts...) if err != nil { return err } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/gocache.go b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/gocache.go index 81b8a8e0..718e63c7 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/gocache.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/gocache.go @@ -58,13 +58,13 @@ func NewNoopCache(ctx context.Context) *NoopCache { return &NoopCache{} } -func (g *NoopCache) Get(req *http.Request) (*http.Response, error) { +func (g *NoopCache) Get(req *http.Request, opts ...CacheOption) (*http.Response, error) { // This isn't threadsafe but who cares? It's the noop cache. g.counter++ return nil, nil } -func (n *NoopCache) Set(req *http.Request, value *http.Response) error { +func (n *NoopCache) Set(req *http.Request, value *http.Response, opts ...CacheOption) error { return nil } @@ -219,12 +219,12 @@ func (g *GoCache) Stats(ctx context.Context) CacheStats { } } -func (g *GoCache) Get(req *http.Request) (*http.Response, error) { +func (g *GoCache) Get(req *http.Request, opts ...CacheOption) (*http.Response, error) { if g.rootLibrary == nil { return nil, nil } - key, err := CreateCacheKey(req) + key, err := CreateCacheKey(req, opts...) if err != nil { return nil, err } @@ -247,12 +247,12 @@ func (g *GoCache) Get(req *http.Request) (*http.Response, error) { return resp, nil } -func (g *GoCache) Set(req *http.Request, value *http.Response) error { +func (g *GoCache) Set(req *http.Request, value *http.Response, opts ...CacheOption) error { if g.rootLibrary == nil { return nil } - key, err := CreateCacheKey(req) + key, err := CreateCacheKey(req, opts...) if err != nil { return err } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go index c9a0abe9..22806159 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/uhttp/wrapper.go @@ -83,6 +83,31 @@ func WithMetricsHandler(handler metrics.Handler) WrapperOption { return metricsHandlerOption{handler: handler} } +type cacheKeyHeadersWrapperOption struct { + opt CacheOption +} + +func (o cacheKeyHeadersWrapperOption) Apply(c *BaseHttpClient) { + c.cacheOptions = append(c.cacheOptions, o.opt) +} + +// WithCacheKeyHeaders returns a WrapperOption that additionally folds the +// named headers into the HTTP response cache key for every request this +// client makes, on top of the default set (Accept, Content-Type, Cookie, +// Range). Use this when requests through this client vary by a header the +// cache wouldn't otherwise key on -- e.g. a per-call Authorization token or +// a tenant/version header -- so requests that only differ in that header +// don't collide in the cache. The value folded in is always read from +// req.Header at request time, so the key can never describe a value other +// than the one actually sent. +// +// Named headers must be set on the request before it reaches Do; a header +// only added later by a transport-level RoundTripper or a cookie jar is not +// seen by the cache lookup and will not be reflected in the key. +func WithCacheKeyHeaders(headers ...string) WrapperOption { + return cacheKeyHeadersWrapperOption{opt: CacheKeyHeaders(headers...)} +} + type WrapperOption interface { Apply(*BaseHttpClient) } @@ -120,6 +145,7 @@ type ( rateLimiter uRateLimit.Limiter baseHttpCache icache metricsHandler metrics.Handler + cacheOptions []CacheOption } DoOption func(resp *WrapperResponse) error @@ -495,7 +521,7 @@ func (c *BaseHttpClient) Do(req *http.Request, options ...DoOption) (*http.Respo } if req.Method == http.MethodGet && req.Header.Get("Cache-Control") != "no-cache" { - resp, err = c.baseHttpCache.Get(req) + resp, err = c.baseHttpCache.Get(req, c.cacheOptions...) if err != nil { return nil, err } @@ -567,7 +593,7 @@ func (c *BaseHttpClient) Do(req *http.Request, options ...DoOption) (*http.Respo } if req.Method == http.MethodGet && resp.StatusCode == http.StatusOK { - cacheErr := c.baseHttpCache.Set(req, resp) + cacheErr := c.baseHttpCache.Set(req, resp, c.cacheOptions...) if cacheErr != nil { l.Warn("error setting cache", zap.String("url", req.URL.String()), zap.Error(cacheErr)) } diff --git a/vendor/modules.txt b/vendor/modules.txt index 8f635dde..85c0e6ec 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -278,7 +278,7 @@ github.com/cockroachdb/swiss # github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 ## explicit; go 1.19 github.com/cockroachdb/tokenbucket -# github.com/conductorone/baton-sdk v0.24.4 +# github.com/conductorone/baton-sdk v0.25.2-0.20260827221151-1ff7ce2d6fda ## explicit; go 1.25.2 github.com/conductorone/baton-sdk/internal/connector github.com/conductorone/baton-sdk/pb/c1/c1z/v1 @@ -312,6 +312,7 @@ github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/internal/rawdb github.com/conductorone/baton-sdk/pkg/dotc1z/format/v3 +github.com/conductorone/baton-sdk/pkg/exit github.com/conductorone/baton-sdk/pkg/field github.com/conductorone/baton-sdk/pkg/healthcheck github.com/conductorone/baton-sdk/pkg/lambda/grpc @@ -325,6 +326,7 @@ github.com/conductorone/baton-sdk/pkg/ratelimit github.com/conductorone/baton-sdk/pkg/retry github.com/conductorone/baton-sdk/pkg/sdk github.com/conductorone/baton-sdk/pkg/session +github.com/conductorone/baton-sdk/pkg/sourcecache github.com/conductorone/baton-sdk/pkg/sync github.com/conductorone/baton-sdk/pkg/sync/expand github.com/conductorone/baton-sdk/pkg/sync/expand/scc From 040d5aca7c2e0abdfb3098792a5067a928636eea Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:43:08 +0000 Subject: [PATCH 36/49] fix(secrets): gate organization API key deletion on its own grant The organization API key deleter was registered whenever sync-secrets was on. Any install already syncing secrets would therefore acquire org-wide Datadog API key deletion the moment it upgraded the connector, without anyone choosing it. Reading a credential inventory is not consent to destroy what is in it. Deletion now needs allow-org-api-key-deletion, a separate flag that defaults to off. Sync behaviour is unchanged either way. The capability has to be absent, not merely refused: C1 resolves what it may do from the advertisement, and the SDK derives CAPABILITY_RESOURCE_DELETE from a type assertion on the registered syncer. So Delete moves off apiTokenBuilder onto deletableAPITokenBuilder and only that variant is registered when the grant is set. The advertised Datadog permissions follow the same split: api_keys_delete and api_keys_write are advertised only by the variant that can reach them. Also updates credential_lifecycle_test.go for the moved constructor. Co-authored-by: c1-squire-dev[bot] --- cmd/baton-datadog/main.go | 9 +- pkg/config/conf.gen.go | 1 + pkg/config/config.go | 11 +++ pkg/connector/api_token.go | 28 ++++++- pkg/connector/connector.go | 24 +++++- pkg/connector/credential_gate_test.go | 98 ++++++++++++++++++++++ pkg/connector/credential_lifecycle_test.go | 42 ++++++---- pkg/connector/resource_types.go | 66 ++++++++------- 8 files changed, 226 insertions(+), 53 deletions(-) create mode 100644 pkg/connector/credential_gate_test.go diff --git a/cmd/baton-datadog/main.go b/cmd/baton-datadog/main.go index 59684761..946e530a 100644 --- a/cmd/baton-datadog/main.go +++ b/cmd/baton-datadog/main.go @@ -20,9 +20,14 @@ func main() { version, cfg.Config, connector.New, + // Every optional surface is forced on so `./connector capabilities` + // documents the connector's full capability set. The generated document + // is static and has no conditional form; the per-install gating lives + // in ResourceSyncers, keyed on the flags below. connectorrunner.WithDefaultCapabilitiesConnectorBuilderV2(&connector.Datadog{ - SyncSecrets: true, - SyncSchedules: true, + SyncSecrets: true, + SyncSchedules: true, + AllowOrgAPIKeyDeletion: true, }), ) } diff --git a/pkg/config/conf.gen.go b/pkg/config/conf.gen.go index 3af4747c..41fe884d 100644 --- a/pkg/config/conf.gen.go +++ b/pkg/config/conf.gen.go @@ -8,6 +8,7 @@ type Datadog struct { ApiKey string `mapstructure:"api-key"` AppKey string `mapstructure:"app-key"` SyncSecrets bool `mapstructure:"sync-secrets"` + AllowOrgApiKeyDeletion bool `mapstructure:"allow-org-api-key-deletion"` SyncSchedules bool `mapstructure:"sync-schedules"` BaseUrl string `mapstructure:"base-url"` } diff --git a/pkg/config/config.go b/pkg/config/config.go index 89b48ed9..1e883c16 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -38,6 +38,16 @@ var ( field.WithDescription("Whether to sync secrets or not"), field.WithDisplayName("Sync secrets"), ) + // AllowOrgAPIKeyDeletion is a destructive grant, deliberately separate from + // SyncSecrets. Reading organization API keys is not consent to destroy + // them, and an install that already syncs secrets must not gain org-wide + // key deletion by upgrading the connector. + AllowOrgAPIKeyDeletion = field.BoolField( + "allow-org-api-key-deletion", + field.WithDescription("Allow this connector to delete Datadog organization API keys. Off by default: syncing secrets does not grant deletion."), + field.WithDefaultValue(false), + field.WithDisplayName("Allow organization API key deletion"), + ) SyncSchedules = field.BoolField( "sync-schedules", field.WithDescription("Whether to sync on-call schedules or not"), @@ -65,6 +75,7 @@ var Config = field.NewConfiguration([]field.SchemaField{ ApiKey, AppKey, SyncSecrets, + AllowOrgAPIKeyDeletion, SyncSchedules, BaseURL, }, diff --git a/pkg/connector/api_token.go b/pkg/connector/api_token.go index 8d44a326..18cfa1e6 100644 --- a/pkg/connector/api_token.go +++ b/pkg/connector/api_token.go @@ -25,10 +25,24 @@ type apiTokenBuilder struct { wrapper *client.DatadogClient } +// deletableAPITokenBuilder is apiTokenBuilder plus the organization API key +// delete path, registered only when the operator sets +// allow-org-api-key-deletion. +// +// Delete lives on this type rather than on apiTokenBuilder because the SDK +// derives CAPABILITY_RESOURCE_DELETE from a type assertion on the registered +// syncer (connectorbuilder.builder.resourceDeleters). A Delete method on +// apiTokenBuilder itself would advertise org-wide key deletion on every +// sync-secrets install regardless of the grant, and a capability that is +// advertised but refuses at call time is worse than one that is absent: C1 +// resolves what it may do from the advertisement, not from the error. +type deletableAPITokenBuilder struct{ *apiTokenBuilder } + var _ connectorbuilder.ResourceSyncerV2 = &apiTokenBuilder{} -var _ connectorbuilder.ResourceDeleterV2Limited = &apiTokenBuilder{} +var _ connectorbuilder.ResourceSyncerV2 = &deletableAPITokenBuilder{} +var _ connectorbuilder.ResourceDeleterV2Limited = &deletableAPITokenBuilder{} -func (o *apiTokenBuilder) Delete(ctx context.Context, resourceID *v2.ResourceId, _ *v2.ResourceId) (annotations.Annotations, error) { +func (o *deletableAPITokenBuilder) Delete(ctx context.Context, resourceID *v2.ResourceId, _ *v2.ResourceId) (annotations.Annotations, error) { if resourceID == nil { return nil, status.Error(codes.InvalidArgument, "baton-datadog: API key id is required") } @@ -159,7 +173,7 @@ func (o *apiTokenBuilder) List( } rv, err := resource.NewSecretResource( name, - apiTokenResourceType, + o.resourceType, *apiToken.Id, options, resourceOptions..., @@ -187,3 +201,11 @@ func newApiTokenBuilder(wrapper *client.DatadogClient) *apiTokenBuilder { wrapper: wrapper, } } + +func newDeletableAPITokenBuilder(wrapper *client.DatadogClient) *deletableAPITokenBuilder { + builder := newApiTokenBuilder(wrapper) + // Same resource type id, but the permission set that includes the delete + // and create rights this variant can actually reach. + builder.resourceType = deletableAPITokenResourceType + return &deletableAPITokenBuilder{apiTokenBuilder: builder} +} diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index 5897e84b..a06ab201 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -30,13 +30,19 @@ type Datadog struct { baseURL string SyncSecrets bool SyncSchedules bool + // AllowOrgAPIKeyDeletion is the operator's explicit grant to destroy + // Datadog organization API keys. It is deliberately not implied by + // SyncSecrets: reading a credential inventory is not consent to delete + // from it, and an install already running with sync-secrets on must not + // acquire org-wide key deletion merely by upgrading the connector. + AllowOrgAPIKeyDeletion bool } // ResourceSyncers returns a ResourceSyncer for each resource type that should be synced from the upstream service. func (d *Datadog) ResourceSyncers(ctx context.Context) []connectorbuilder.ResourceSyncerV2 { userSyncer := connectorbuilder.ResourceSyncerV2(newUserBuilder(d.wrapper)) if d.SyncSecrets { - userSyncer = newCredentialUserBuilder(d.wrapper) + userSyncer = newCredentialUserBuilder(d.wrapper, d.AllowOrgAPIKeyDeletion) } resourceSyncers := []connectorbuilder.ResourceSyncerV2{ userSyncer, @@ -45,7 +51,15 @@ func (d *Datadog) ResourceSyncers(ctx context.Context) []connectorbuilder.Resour } if d.SyncSecrets { - resourceSyncers = append(resourceSyncers, newApiTokenBuilder(d.wrapper), newApplicationKeyBuilder(d.wrapper)) + // The organization API key syncer is registered either way; only the + // variant carrying Delete is gated, so CAPABILITY_RESOURCE_DELETE is + // absent from the advertised capabilities without the grant rather + // than advertised and refused. + apiTokenSyncer := connectorbuilder.ResourceSyncerV2(newApiTokenBuilder(d.wrapper)) + if d.AllowOrgAPIKeyDeletion { + apiTokenSyncer = newDeletableAPITokenBuilder(d.wrapper) + } + resourceSyncers = append(resourceSyncers, apiTokenSyncer, newApplicationKeyBuilder(d.wrapper)) } if d.SyncSchedules { @@ -137,6 +151,7 @@ func New(ctx context.Context, ddc *cfg.Datadog, _ *cli.ConnectorOpts) (connector baseURL := ddc.BaseUrl syncSecrets := ddc.SyncSecrets syncSchedules := ddc.SyncSchedules + allowOrgAPIKeyDeletion := ddc.AllowOrgApiKeyDeletion // Validate input parameters if site == "" { @@ -185,7 +200,8 @@ func New(ctx context.Context, ddc *cfg.Datadog, _ *cli.ConnectorOpts) (connector baseURL: baseURL, client: officialClient, wrapper: wrapper, - SyncSecrets: syncSecrets, - SyncSchedules: syncSchedules, + SyncSecrets: syncSecrets, + SyncSchedules: syncSchedules, + AllowOrgAPIKeyDeletion: allowOrgAPIKeyDeletion, }, nil, nil } diff --git a/pkg/connector/credential_gate_test.go b/pkg/connector/credential_gate_test.go new file mode 100644 index 00000000..8178e646 --- /dev/null +++ b/pkg/connector/credential_gate_test.go @@ -0,0 +1,98 @@ +package connector + +import ( + "context" + "testing" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/stretchr/testify/require" +) + +// newGateTestConnector builds the connector the way New would, but against a +// fake provider, so a test can read the capabilities C1 would actually be +// advertised for a given flag combination. +func newGateTestConnector(serverURL string, syncSecrets, allowOrgAPIKeyDeletion bool) *Datadog { + return &Datadog{ + wrapper: newLifecycleTestWrapper(serverURL), + site: "example.com", + apiKey: "connector-api-key", + appKey: "connector-app-key", + SyncSecrets: syncSecrets, + AllowOrgAPIKeyDeletion: allowOrgAPIKeyDeletion, + } +} + +// resourceTypeCapabilities returns the capabilities the SDK advertises for one +// resource type id, going through NewConnector/GetMetadata rather than +// inspecting the builders directly: the whole point of the gate is what C1 +// sees in the advertisement, not what a method exists to do. +func resourceTypeCapabilities(t *testing.T, d *Datadog, resourceTypeID string) []v2.Capability { + t.Helper() + ctx := context.Background() + server, err := connectorbuilder.NewConnector(ctx, d) + require.NoError(t, err) + md, err := server.GetMetadata(ctx, &v2.ConnectorServiceGetMetadataRequest{}) + require.NoError(t, err) + for _, rtc := range md.GetMetadata().GetCapabilities().GetResourceTypeCapabilities() { + if rtc.GetResourceType().GetId() == resourceTypeID { + return rtc.GetCapabilities() + } + } + t.Fatalf("resource type %q was not advertised at all", resourceTypeID) + return nil +} + +// TestOrgAPIKeyDeleteRequiresItsOwnGrant is the regression this gate exists +// for: an install already running with sync-secrets on must not acquire +// org-wide Datadog API key deletion by upgrading the connector. +func TestOrgAPIKeyDeleteRequiresItsOwnGrant(t *testing.T) { + d := newGateTestConnector("http://127.0.0.1:1", true, false) + caps := resourceTypeCapabilities(t, d, apiTokenResourceType.Id) + require.Contains(t, caps, v2.Capability_CAPABILITY_SYNC, + "sync-secrets alone must still sync organization API keys") + require.NotContains(t, caps, v2.Capability_CAPABILITY_RESOURCE_DELETE, + "sync-secrets alone must not advertise organization API key deletion") +} + +func TestOrgAPIKeyDeleteAdvertisedWithGrant(t *testing.T) { + d := newGateTestConnector("http://127.0.0.1:1", true, true) + caps := resourceTypeCapabilities(t, d, apiTokenResourceType.Id) + require.Contains(t, caps, v2.Capability_CAPABILITY_RESOURCE_DELETE) +} + +// TestOrgAPIKeyDeletePermissionFollowsTheGrant checks the advertised Datadog +// permissions track the advertised capabilities: an install that cannot delete +// must not be told to grant api_keys_delete. +func TestOrgAPIKeyDeletePermissionFollowsTheGrant(t *testing.T) { + ctx := context.Background() + for _, tt := range []struct { + name string + granted bool + want []string + absent []string + }{ + {name: "grant off", granted: false, want: []string{"api_keys_read"}, absent: []string{"api_keys_delete", "api_keys_write"}}, + {name: "grant on", granted: true, want: []string{"api_keys_read", "api_keys_write", "api_keys_delete"}}, + } { + t.Run(tt.name, func(t *testing.T) { + server, err := connectorbuilder.NewConnector(ctx, newGateTestConnector("http://127.0.0.1:1", true, tt.granted)) + require.NoError(t, err) + md, err := server.GetMetadata(ctx, &v2.ConnectorServiceGetMetadataRequest{}) + require.NoError(t, err) + var got []string + for _, rtc := range md.GetMetadata().GetCapabilities().GetResourceTypeCapabilities() { + if rtc.GetResourceType().GetId() != apiTokenResourceType.Id { + continue + } + for _, p := range rtc.GetPermissions().GetPermissions() { + got = append(got, p.GetPermission()) + } + } + require.ElementsMatch(t, tt.want, got) + for _, absent := range tt.absent { + require.NotContains(t, got, absent) + } + }) + } +} diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index c644e5e2..f14a69cf 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -134,7 +134,7 @@ func TestApiTokenBuilderDeleteUsesHandleNotSecret(t *testing.T) { require.Equal(t, handle, issued.ID) require.Equal(t, secret, issued.Secret) - deleter := newApiTokenBuilder(wrapper) + deleter := newDeletableAPITokenBuilder(wrapper) resourceID := &v2.ResourceId{ResourceType: apiTokenResourceType.Id, Resource: issued.ID} _, err = deleter.Delete(ctx, resourceID, nil) require.NoError(t, err) @@ -185,7 +185,7 @@ func TestApiTokenBuilderDeleteRejectsMissingHandle(t *testing.T) { defer server.Close() wrapper := newLifecycleTestWrapper(server.URL) - deleter := newApiTokenBuilder(wrapper) + deleter := newDeletableAPITokenBuilder(wrapper) _, err := deleter.Delete(context.Background(), tt.resourceID, nil) require.Error(t, err) require.Equal(t, codes.InvalidArgument, status.Code(err)) @@ -239,11 +239,14 @@ func newServiceAccountAppKeyServer(t *testing.T, serviceAccountID, handle, secre func issueServiceAccountAppKey(t *testing.T, ctx context.Context, wrapper *client.DatadogClient, serviceAccountID, requestID string) *connectorbuilder.CredentialIssueOutput { t.Helper() - issuer := newCredentialUserBuilder(wrapper) + issuer := newCredentialUserBuilder(wrapper, false) out, err := issuer.Issue(ctx, &connectorbuilder.CredentialIssueInput{ - IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: serviceAccountID}, - RequestID: requestID, - CredentialOptions: v2.CredentialIssueOptions_builder{ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build()}.Build(), + IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: serviceAccountID}, + RequestID: requestID, + CredentialOptions: v2.CredentialIssueOptions_builder{ + SecretResourceTypeId: serviceAccountApplicationKeyResourceType.Id, + ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build(), + }.Build(), }) require.NoError(t, err) return out @@ -288,11 +291,14 @@ func TestIssueRequiresServiceAccount(t *testing.T) { defer server.Close() wrapper := newLifecycleTestWrapper(server.URL) - issuer := newCredentialUserBuilder(wrapper) + issuer := newCredentialUserBuilder(wrapper, false) out, err := issuer.Issue(context.Background(), &connectorbuilder.CredentialIssueInput{ - IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: humanUserID}, - RequestID: "req-reject", - CredentialOptions: v2.CredentialIssueOptions_builder{ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build()}.Build(), + IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: humanUserID}, + RequestID: "req-reject", + CredentialOptions: v2.CredentialIssueOptions_builder{ + SecretResourceTypeId: serviceAccountApplicationKeyResourceType.Id, + ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build(), + }.Build(), }) require.Nil(t, out) require.Error(t, err) @@ -330,11 +336,14 @@ func TestIssueRefusesDuplicateRequest(t *testing.T) { defer server.Close() wrapper := newLifecycleTestWrapper(server.URL) - issuer := newCredentialUserBuilder(wrapper) + issuer := newCredentialUserBuilder(wrapper, false) out, err := issuer.Issue(context.Background(), &connectorbuilder.CredentialIssueInput{ - IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID}, - RequestID: requestID, - CredentialOptions: v2.CredentialIssueOptions_builder{ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build()}.Build(), + IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID}, + RequestID: requestID, + CredentialOptions: v2.CredentialIssueOptions_builder{ + SecretResourceTypeId: serviceAccountApplicationKeyResourceType.Id, + ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build(), + }.Build(), }) require.Nil(t, out) require.Error(t, err) @@ -976,12 +985,13 @@ func TestIssuePassesScopesToProviderAndProfile(t *testing.T) { })) defer server.Close() - issuer := newCredentialUserBuilder(newLifecycleTestWrapper(server.URL)) + issuer := newCredentialUserBuilder(newLifecycleTestWrapper(server.URL), false) out, err := issuer.Issue(context.Background(), &connectorbuilder.CredentialIssueInput{ IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID}, RequestID: "req-scoped", CredentialOptions: v2.CredentialIssueOptions_builder{ - ApiKey: v2.CredentialIssueOptions_ApiKey_builder{Scopes: requested}.Build(), + SecretResourceTypeId: serviceAccountApplicationKeyResourceType.Id, + ApiKey: v2.CredentialIssueOptions_ApiKey_builder{Scopes: requested}.Build(), }.Build(), }) require.NoError(t, err) diff --git a/pkg/connector/resource_types.go b/pkg/connector/resource_types.go index 94f5fa64..8eaba058 100644 --- a/pkg/connector/resource_types.go +++ b/pkg/connector/resource_types.go @@ -68,36 +68,30 @@ var ( } // apiTokenResourceType covers organization-scoped API keys (Datadog's // "API keys", /api/v2/api_keys): org-wide credentials not owned by any - // single Datadog identity. This connector still syncs and can delete - // them, but Issue no longer targets this type -- an org-scoped key - // issued on behalf of a selected user is not an honest mapping of who - // holds it. See serviceAccountApplicationKeyResourceType for the type - // Issue does target. + // single Datadog identity. Issue targets this type only when the operator + // grants allow-org-api-key-deletion -- an org-scoped key has no honest + // owner, and the SDK will not advertise an issuance option whose secret + // resource type has no revoke path. See + // serviceAccountApplicationKeyResourceType for the type Issue prefers. // // The advertised permissions are the ones Datadog's own API spec marks - // required (the per-operation "x-permission" block) for the only two - // endpoints the advertised capabilities call: ListAPIKeys, backing - // CAPABILITY_SYNC, requires api_keys_read, and DeleteAPIKey - // (DELETE /api/v2/api_keys/{api_key_id}), backing - // CAPABILITY_RESOURCE_DELETE, requires api_keys_delete. api_keys_delete - // is a real Datadog permission ("API Keys Delete -- Delete API Keys for - // your organization", Datadog Admin Role) and is the one that governs - // delete; api_keys_write is scoped to CreateAPIKey/UpdateAPIKey ("Create - // and rename API Keys") and is deliberately NOT advertised here, because - // no advertised capability on this type creates or renames a key. - // Advertising it would make C1 demand org-wide key-creation rights the - // connector never exercises. CreateAPIKey/FindAPIKeyByName remain on - // DatadogClient for tests. - apiTokenResourceType = &v2.ResourceType{ - Id: "api-key", - DisplayName: "Organization API Key", - Description: "A Datadog organization API key. Owned by the org, not by any single Datadog identity; not used for credential issuance by this connector.", - Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_SECRET}, - Annotations: annotations.New( - &v2.SkipEntitlementsAndGrants{}, - capabilityPermissions("api_keys_read", "api_keys_delete"), - ), - } + // required (the per-operation "x-permission" block) for the endpoints the + // advertised capabilities call. api_keys_read backs CAPABILITY_SYNC + // (ListAPIKeys) and is always advertised. api_keys_write backs CreateAPIKey + // and so is advertised only by deletableAPITokenResourceType, which is also + // the only variant that can be an issuance target. api_keys_delete backs + // CAPABILITY_RESOURCE_DELETE (DeleteAPIKey) and is likewise only advertised + // on that variant: this base type carries no delete path, so demanding a + // Datadog Admin permission for it would send operators after rights no + // reachable code exercises. + apiTokenResourceType = newAPITokenResourceType("api_keys_read") + // deletableAPITokenResourceType is apiTokenResourceType as registered when + // allow-org-api-key-deletion is on: same id, plus the permissions the + // delete and issue paths that grant unlocks actually require. + // api_keys_delete is "API Keys Delete -- Delete API Keys for your + // organization" and api_keys_write is "Create and rename API Keys", both + // Datadog Admin Role permissions. + deletableAPITokenResourceType = newAPITokenResourceType("api_keys_read", "api_keys_write", "api_keys_delete") // serviceAccountApplicationKeyResourceType covers application keys owned // by a Datadog service-account user (/api/v2/service_accounts/{id}/application_keys). // This is the resource type credential issuance targets: the key is @@ -151,3 +145,19 @@ var ( ), } ) + +// newAPITokenResourceType builds the organization API key resource type with a +// given permission set. The id is shared across variants because only one of +// them is ever registered: which one depends on allow-org-api-key-deletion. +func newAPITokenResourceType(permissions ...string) *v2.ResourceType { + return &v2.ResourceType{ + Id: "api-key", + DisplayName: "Organization API Key", + Description: "A Datadog organization API key. Owned by the org, not by any single Datadog identity.", + Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_SECRET}, + Annotations: annotations.New( + &v2.SkipEntitlementsAndGrants{}, + capabilityPermissions(permissions...), + ), + } +} From 4edb89344045840947b1b4d81c0245c5793c1339 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:43:18 +0000 Subject: [PATCH 37/49] feat(secrets): issue both Datadog API key kinds, chosen by the caller Datadog has two kinds of API key and they are not interchangeable. A service account application key is scoped to and owned by one identity; an organization API key is owned by the whole organization, has no owner inside it, and cannot be scoped at all. Both are the API_KEY shape, so the shape enum alone cannot tell a caller which one it is getting. IssueCapabilityDetails now advertises one descriptor per kind, separated by secret_resource_type_id, with the service account application key marked preferred. Issue dispatches on the requested type rather than always minting an application key, and refuses an unrecognised type instead of falling back to a default arm. Organization API key issuance follows allow-org-api-key-deletion. The SDK will not register an issuance descriptor whose secret resource type has no ResourceDeleterV2, and that is the right constraint: a credential this connector cannot revoke is one it should not mint. Co-authored-by: c1-squire-dev[bot] --- pkg/connector/credential_issue_kinds_test.go | 153 ++++++++++++++++++ pkg/connector/credential_smoke_test.go | 9 +- pkg/connector/users.go | 160 +++++++++++++++++-- 3 files changed, 304 insertions(+), 18 deletions(-) create mode 100644 pkg/connector/credential_issue_kinds_test.go diff --git a/pkg/connector/credential_issue_kinds_test.go b/pkg/connector/credential_issue_kinds_test.go new file mode 100644 index 00000000..3dcea77b --- /dev/null +++ b/pkg/connector/credential_issue_kinds_test.go @@ -0,0 +1,153 @@ +package connector + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "sync" + "testing" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// TestIssuanceAdvertisesBothCredentialKinds is the type-discriminator contract: +// two kinds of the same API_KEY shape, told apart only by +// secret_resource_type_id. +func TestIssuanceAdvertisesBothCredentialKinds(t *testing.T) { + ctx := context.Background() + details, _, err := newCredentialUserBuilder(newLifecycleTestWrapper("http://127.0.0.1:1"), true).IssueCapabilityDetails(ctx) + require.NoError(t, err) + require.Len(t, details.GetOptions(), 2) + + byType := map[string]*v2.CredentialIssueOptionDescriptor{} + for _, o := range details.GetOptions() { + require.Equal(t, v2.CapabilityDetailCredentialOption_CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY, o.GetOption(), + "both kinds are the same shape; only the secret resource type separates them") + byType[o.GetSecretResourceTypeId()] = o + } + require.Contains(t, byType, serviceAccountApplicationKeyResourceType.Id) + require.Contains(t, byType, apiTokenResourceType.Id) + require.True(t, byType[serviceAccountApplicationKeyResourceType.Id].GetPreferred()) + require.False(t, byType[apiTokenResourceType.Id].GetPreferred()) + require.True(t, byType[serviceAccountApplicationKeyResourceType.Id].GetCustomScopesAllowed()) + require.False(t, byType[apiTokenResourceType.Id].GetCustomScopesAllowed(), + "organization API keys carry no scopes") +} + +// TestIssuanceOmitsOrgAPIKeyWithoutGrant: without the delete grant there is no +// revoke path for an organization API key, and the SDK refuses to register a +// descriptor whose secret resource type has no deleter. Advertising it anyway +// would fail connector startup, so the descriptor has to be absent too. +func TestIssuanceOmitsOrgAPIKeyWithoutGrant(t *testing.T) { + ctx := context.Background() + details, _, err := newCredentialUserBuilder(newLifecycleTestWrapper("http://127.0.0.1:1"), false).IssueCapabilityDetails(ctx) + require.NoError(t, err) + require.Len(t, details.GetOptions(), 1) + require.Equal(t, serviceAccountApplicationKeyResourceType.Id, details.GetOptions()[0].GetSecretResourceTypeId()) + + out, err := newCredentialUserBuilder(newLifecycleTestWrapper("http://127.0.0.1:1"), false).Issue(ctx, &connectorbuilder.CredentialIssueInput{ + IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID}, + RequestID: "req-org-denied", + CredentialOptions: v2.CredentialIssueOptions_builder{ + SecretResourceTypeId: apiTokenResourceType.Id, + ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build(), + }.Build(), + }) + require.Nil(t, out) + require.Equal(t, codes.FailedPrecondition, status.Code(err)) +} + +// TestIssueDispatchesOnRequestedCredentialKind proves the Issue path routes on +// the requested type rather than hardcoding one arm: the same identity and the +// same API_KEY shape must reach a different Datadog endpoint per kind. +func TestIssueDispatchesOnRequestedCredentialKind(t *testing.T) { + ctx := context.Background() + var requests []*recordedRequest + var mu sync.Mutex + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + requests = append(requests, &recordedRequest{method: r.Method, path: r.URL.Path}) + mu.Unlock() + w.Header().Set("Content-Type", "application/json") + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/api_keys": + _, _ = w.Write([]byte(`{"data":[]}`)) + case r.Method == http.MethodPost && r.URL.Path == "/api/v2/api_keys": + _ = json.NewEncoder(w).Encode(map[string]any{"data": map[string]any{ + "id": "org-key-id", "type": "api_keys", + "attributes": map[string]any{"name": "c1-req-org", "key": "org-key-secret"}, + }}) + default: + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":["unexpected"]}`)) + } + })) + defer server.Close() + + issuer := newCredentialUserBuilder(newLifecycleTestWrapper(server.URL), true) + out, err := issuer.Issue(ctx, &connectorbuilder.CredentialIssueInput{ + IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID}, + RequestID: "req-org", + CredentialOptions: v2.CredentialIssueOptions_builder{ + SecretResourceTypeId: apiTokenResourceType.Id, + ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build(), + }.Build(), + }) + require.NoError(t, err) + require.Equal(t, apiTokenResourceType.Id, out.Secret.GetId().GetResourceType(), + "the issued resource must come back as the kind that was requested") + require.Equal(t, "org-key-id", out.Secret.GetId().GetResource()) + require.Len(t, out.PlaintextData, 1) + require.Equal(t, "api_key", out.PlaintextData[0].GetName()) + + mu.Lock() + defer mu.Unlock() + var sawOrgCreate bool + for _, req := range requests { + require.NotContains(t, req.path, "/service_accounts/", + "an organization API key request must never reach the service-account application key API") + if req.method == http.MethodPost && req.path == "/api/v2/api_keys" { + sawOrgCreate = true + } + } + require.True(t, sawOrgCreate, "the organization API key arm must call POST /api/v2/api_keys") +} + +// TestIssueRejectsScopesOnOrgAPIKey: the shape allows scopes, this kind does +// not, so the arm has to fail closed rather than mint an unscoped key. +func TestIssueRejectsScopesOnOrgAPIKey(t *testing.T) { + issuer := newCredentialUserBuilder(newLifecycleTestWrapper("http://127.0.0.1:1"), true) + out, err := issuer.Issue(context.Background(), &connectorbuilder.CredentialIssueInput{ + IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID}, + RequestID: "req-org-scoped", + CredentialOptions: v2.CredentialIssueOptions_builder{ + SecretResourceTypeId: apiTokenResourceType.Id, + ApiKey: v2.CredentialIssueOptions_ApiKey_builder{Scopes: []string{"dashboards_read"}}.Build(), + }.Build(), + }) + require.Nil(t, out) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} + +// TestIssueRejectsUnknownCredentialKind: an unadvertised secret resource type +// is a protocol mismatch, not a cue to fall back to the preferred arm. +func TestIssueRejectsUnknownCredentialKind(t *testing.T) { + issuer := newCredentialUserBuilder(newLifecycleTestWrapper("http://127.0.0.1:1"), true) + for _, secretType := range []string{"", "not-a-datadog-credential"} { + out, err := issuer.Issue(context.Background(), &connectorbuilder.CredentialIssueInput{ + IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID}, + RequestID: "req-unknown", + CredentialOptions: v2.CredentialIssueOptions_builder{ + SecretResourceTypeId: secretType, + ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build(), + }.Build(), + }) + require.Nil(t, out) + require.Equal(t, codes.InvalidArgument, status.Code(err)) + } +} diff --git a/pkg/connector/credential_smoke_test.go b/pkg/connector/credential_smoke_test.go index 4aea63f8..f3239e4d 100644 --- a/pkg/connector/credential_smoke_test.go +++ b/pkg/connector/credential_smoke_test.go @@ -51,7 +51,7 @@ func TestCredentialIssueLifecycle(t *testing.T) { datadogConnector, ok := builder.(*Datadog) require.True(t, ok) - issuer := newCredentialUserBuilder(datadogConnector.wrapper) + issuer := newCredentialUserBuilder(datadogConnector.wrapper, false) requestID := "smoke-" + time.Now().UTC().Format("20060102T150405") t.Logf("issuing Datadog service account application key with request id %q", requestID) issued, err := issuer.Issue(ctx, &connectorbuilder.CredentialIssueInput{ @@ -59,8 +59,11 @@ func TestCredentialIssueLifecycle(t *testing.T) { ResourceType: userResourceType.Id, Resource: serviceAccountID, }, - RequestID: requestID, - CredentialOptions: v2.CredentialIssueOptions_builder{ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build()}.Build(), + RequestID: requestID, + CredentialOptions: v2.CredentialIssueOptions_builder{ + SecretResourceTypeId: serviceAccountApplicationKeyResourceType.Id, + ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build(), + }.Build(), }) require.NoError(t, err) revoked := false diff --git a/pkg/connector/users.go b/pkg/connector/users.go index 8b4d156a..755a0bd8 100644 --- a/pkg/connector/users.go +++ b/pkg/connector/users.go @@ -23,36 +23,103 @@ type userBuilder struct { wrapper *client.DatadogClient } -type credentialUserBuilder struct{ *userBuilder } +type credentialUserBuilder struct { + *userBuilder + // offerOrgAPIKey advertises organization API keys as a second issuance + // kind. It follows the allow-org-api-key-deletion grant because the SDK + // refuses to register an issuance descriptor whose secret resource type + // has no ResourceDeleterV2: without the grant this connector cannot revoke + // an org key, so it must not mint one either. + offerOrgAPIKey bool +} -func newCredentialUserBuilder(wrapper *client.DatadogClient) *credentialUserBuilder { - return &credentialUserBuilder{userBuilder: newUserBuilder(wrapper)} +func newCredentialUserBuilder(wrapper *client.DatadogClient, offerOrgAPIKey bool) *credentialUserBuilder { + return &credentialUserBuilder{userBuilder: newUserBuilder(wrapper), offerOrgAPIKey: offerOrgAPIKey} } +// IssueCapabilityDetails advertises the credential kinds this connector mints. +// Both are the API_KEY shape and they are distinguished only by +// secret_resource_type_id, which is what that field is for: the closed Option +// enum names the shape a caller asks for, the open resource type id names the +// kind that comes back. Datadog's two kinds are genuinely different +// credentials -- a service-account application key is scoped to and owned by +// one identity, an organization API key is owned by the org and by nobody in +// it -- so they must stay two descriptors. Collapsing them would leave a +// caller unable to say which one it wants, and the SDK unable to check that +// the resource Issue returns is the one that was requested. +// +// The service-account application key is marked preferred: it is the only +// issuance mapping with an honest owner, so it is the default when a caller +// asks for the API_KEY shape without choosing a kind. func (u *credentialUserBuilder) IssueCapabilityDetails(context.Context) (*v2.CredentialDetailsCredentialIssue, annotations.Annotations, error) { - return v2.CredentialDetailsCredentialIssue_builder{ - Options: []*v2.CredentialIssueOptionDescriptor{v2.CredentialIssueOptionDescriptor_builder{ + options := []*v2.CredentialIssueOptionDescriptor{ + v2.CredentialIssueOptionDescriptor_builder{ Option: v2.CapabilityDetailCredentialOption_CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY, ResourceMode: v2.CredentialResourceMode_CREDENTIAL_RESOURCE_MODE_DISCOVERABLE, SecretResourceTypeId: serviceAccountApplicationKeyResourceType.Id, CustomScopesAllowed: true, - }.Build()}, + Preferred: true, + }.Build(), + } + if u.offerOrgAPIKey { + options = append(options, v2.CredentialIssueOptionDescriptor_builder{ + Option: v2.CapabilityDetailCredentialOption_CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY, + ResourceMode: v2.CredentialResourceMode_CREDENTIAL_RESOURCE_MODE_DISCOVERABLE, + // deletableAPITokenResourceType, not apiTokenResourceType: they + // share an id, and this is the variant registered whenever this + // descriptor is advertised. + SecretResourceTypeId: deletableAPITokenResourceType.Id, + // Datadog organization API keys carry no scopes. Advertising none + // and disallowing custom ones makes the SDK reject a scoped + // request for this kind before it reaches Issue. + }.Build()) + } + return v2.CredentialDetailsCredentialIssue_builder{ + Options: options, PreferredOption: v2.CapabilityDetailCredentialOption_CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY, }.Build(), nil, nil } -// Issue mints a Datadog service-account application key scoped to and owned -// by the target identity. Per SPEC-07 (the judged Datadog credential-issuance -// design), this is the only honest issuance mapping this connector supports: -// an organization API key or a current-user application key has no reliable -// non-human owner, so Issue targets a service-account application key -// instead, gated on a live re-check that the target is actually a Datadog -// service account (its user record may have changed since it was last -// synced). +// Issue dispatches on the credential kind the caller selected. The oneof arm +// of CredentialIssueOptions gives the shape (API_KEY) and +// secret_resource_type_id gives the kind within it; the SDK has already +// resolved that pair against IssueCapabilityDetails and rejected anything not +// advertised, so this switch only has to route. It deliberately does not fall +// back to a default arm: an unrecognised kind is a protocol mismatch, and +// minting the wrong kind of Datadog credential is not a recoverable guess. func (u *credentialUserBuilder) Issue(ctx context.Context, input *connectorbuilder.CredentialIssueInput) (*connectorbuilder.CredentialIssueOutput, error) { if input == nil || input.IdentityID == nil || input.IdentityID.GetResourceType() != userResourceType.Id { return nil, status.Error(codes.InvalidArgument, "baton-datadog: a Datadog user identity is required") } + switch secretResourceTypeID := input.CredentialOptions.GetSecretResourceTypeId(); secretResourceTypeID { + case serviceAccountApplicationKeyResourceType.Id: + return u.issueServiceAccountApplicationKey(ctx, input) + case apiTokenResourceType.Id: + if !u.offerOrgAPIKey { + return nil, status.Error(codes.FailedPrecondition, + "baton-datadog: organization API key issuance requires allow-org-api-key-deletion, which also provides the revoke path") + } + return u.issueOrganizationAPIKey(ctx, input) + default: + return nil, status.Errorf(codes.InvalidArgument, + "baton-datadog: unsupported credential secret resource type %q", secretResourceTypeID) + } +} + +// issuedCredentialName is the provider-side name this connector gives a +// credential it mints. It is derived from the request id so a retried request +// finds the key its predecessor created instead of minting a second one. +func issuedCredentialName(requestID string) string { + return "c1-" + requestID +} + +// issueServiceAccountApplicationKey mints a Datadog service-account +// application key scoped to and owned by the target identity. Per SPEC-07 (the +// judged Datadog credential-issuance design) this is the honest issuance +// mapping: the key has a real non-human owner. It is gated on a live re-check +// that the target is actually a Datadog service account, since its user record +// may have changed since it was last synced. +func (u *credentialUserBuilder) issueServiceAccountApplicationKey(ctx context.Context, input *connectorbuilder.CredentialIssueInput) (*connectorbuilder.CredentialIssueOutput, error) { serviceAccountID := input.IdentityID.GetResource() userResp, err := u.wrapper.GetUser(ctx, serviceAccountID) @@ -66,7 +133,7 @@ func (u *credentialUserBuilder) Issue(ctx context.Context, input *connectorbuild return nil, status.Errorf(codes.InvalidArgument, "baton-datadog: Datadog user %q is not a service account; credential issuance only targets service accounts", serviceAccountID) } - name := "c1-" + input.RequestID + name := issuedCredentialName(input.RequestID) existing, err := u.wrapper.FindServiceAccountApplicationKeyByName(ctx, serviceAccountID, name) if err != nil { return nil, fmt.Errorf("baton-datadog: look up application key for request %q: %w", input.RequestID, err) @@ -114,6 +181,69 @@ func (u *credentialUserBuilder) Issue(ctx context.Context, input *connectorbuild }, nil } +// issueOrganizationAPIKey mints a Datadog organization API key +// (POST /api/v2/api_keys). This is the second, deliberately non-default +// issuance kind: the key is org-wide and Datadog records its creator, not an +// owner, so the identity on the returned SecretTrait is the identity the key +// was vended TO rather than a provider-side owner Datadog would enforce. The +// key is unscoped -- a Datadog organization API key carries no scopes at all, +// which is exactly why it must stay a separate kind from a service-account +// application key rather than a variation of one. +func (u *credentialUserBuilder) issueOrganizationAPIKey(ctx context.Context, input *connectorbuilder.CredentialIssueInput) (*connectorbuilder.CredentialIssueOutput, error) { + // The SDK rejects requested scopes for this descriptor before Issue runs + // (it advertises no scopes and disallows custom ones). Re-checking here + // keeps the arm correct for a caller that reaches Issue directly, and + // fails closed rather than silently minting an unscoped key for a request + // that asked for a scoped one. + if scopes := input.CredentialOptions.GetApiKey().GetScopes(); len(scopes) != 0 { + return nil, status.Error(codes.InvalidArgument, + "baton-datadog: Datadog organization API keys cannot be scoped; request a service account application key for a scoped credential") + } + + name := issuedCredentialName(input.RequestID) + existing, err := u.wrapper.FindAPIKeyByName(ctx, name) + if err != nil { + return nil, fmt.Errorf("baton-datadog: look up organization API key for request %q: %w", input.RequestID, err) + } + if existing != nil { + return nil, status.Errorf(codes.AlreadyExists, "baton-datadog: organization API key for request %q may already exist; refusing to issue a duplicate", input.RequestID) + } + + key, err := u.wrapper.CreateAPIKey(ctx, name) + if err != nil { + return nil, fmt.Errorf("baton-datadog: create organization API key: %w", err) + } + + secretTraitOptions := []rs.SecretTraitOption{ + rs.WithSecretCreatedByID(input.IdentityID), + rs.WithSecretIdentityID(input.IdentityID), + rs.WithSecretType(v2.SecretTrait_CREDENTIAL_TYPE_STATIC_SECRET), + rs.WithSecretDetail("datadog.api_key"), + } + // No parent resource id: an organization API key hangs off the + // organization, not off the identity it was vended to, and the syncer + // (apiTokenBuilder.List) builds these keys without a parent too. Claiming + // the user as a parent here would make the issued resource disagree with + // the same key on its next sync. + secret, err := rs.NewSecretResource(name, deletableAPITokenResourceType, key.ID, secretTraitOptions) + if err != nil { + if deleteErr := u.wrapper.DeleteAPIKey(ctx, key.ID); deleteErr != nil { + ctxzap.Extract(ctx).Warn("failed to clean up Datadog organization API key after resource construction error", + zap.String("api_key_id", key.ID), + zap.Error(deleteErr), + ) + } + return nil, fmt.Errorf("baton-datadog: build organization API key secret resource: %w", err) + } + return &connectorbuilder.CredentialIssueOutput{ + Secret: secret, + PlaintextData: []*v2.PlaintextData{ + v2.PlaintextData_builder{Name: "api_key", Bytes: []byte(key.Secret)}.Build(), + }, + ResourceMode: v2.CredentialResourceMode_CREDENTIAL_RESOURCE_MODE_DISCOVERABLE, + }, nil +} + var _ connectorbuilder.ResourceSyncerV2 = &userBuilder{} var _ connectorbuilder.AccountManagerV2 = &userBuilder{} var _ connectorbuilder.ResourceActionProvider = &userBuilder{} From d7168867286d4a31b45fdaf879749679b2d7267c Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:43:18 +0000 Subject: [PATCH 38/49] docs: regenerate connector metadata for the new flag and key kinds Regenerated from the built connector: ./connector config > config_schema.json ./connector capabilities > baton_capabilities.json README and docs/connector.mdx describe allow-org-api-key-deletion and the two issuance kinds. The capabilities document is static and has no conditional form, so main.go forces every optional surface on when generating it, as it already did for sync-secrets and sync-schedules. Co-authored-by: c1-squire-dev[bot] --- README.md | 1 + baton_capabilities.json | 16 ++++++++++++++-- config_schema.json | 6 ++++++ docs/connector.mdx | 15 ++++++++++++--- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 8d5d2491..53af09d8 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,7 @@ Available Commands: help Help about any command Flags: + --allow-org-api-key-deletion Allow this connector to delete Datadog organization API keys. Off by default: syncing secrets does not grant deletion. ($BATON_ALLOW_ORG_API_KEY_DELETION) --api-key string required: API key used to authenticate to Datadog API. ($BATON_API_KEY) --app-key string required: APP key used with API key to assign scopes for API access. ($BATON_APP_KEY) --client-id string The client ID used to authenticate with ConductorOne ($BATON_CLIENT_ID) diff --git a/baton_capabilities.json b/baton_capabilities.json index 32249a6e..88b0f6e0 100644 --- a/baton_capabilities.json +++ b/baton_capabilities.json @@ -18,13 +18,16 @@ { "permission": "api_keys_read" }, + { + "permission": "api_keys_write" + }, { "permission": "api_keys_delete" } ] } ], - "description": "A Datadog organization API key. Owned by the org, not by any single Datadog identity; not used for credential issuance by this connector." + "description": "A Datadog organization API key. Owned by the org, not by any single Datadog identity." }, "capabilities": [ "CAPABILITY_SYNC", @@ -35,6 +38,9 @@ { "permission": "api_keys_read" }, + { + "permission": "api_keys_write" + }, { "permission": "api_keys_delete" } @@ -209,7 +215,13 @@ "option": "CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY", "customScopesAllowed": true, "resourceMode": "CREDENTIAL_RESOURCE_MODE_DISCOVERABLE", - "secretResourceTypeId": "service-account-application-key" + "secretResourceTypeId": "service-account-application-key", + "preferred": true + }, + { + "option": "CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY", + "resourceMode": "CREDENTIAL_RESOURCE_MODE_DISCOVERABLE", + "secretResourceTypeId": "api-key" } ], "preferredOption": "CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY" diff --git a/config_schema.json b/config_schema.json index a8daf003..01080e57 100644 --- a/config_schema.json +++ b/config_schema.json @@ -141,6 +141,12 @@ "description": "Whether to sync secrets or not", "boolField": {} }, + { + "name": "allow-org-api-key-deletion", + "displayName": "Allow organization API key deletion", + "description": "Allow this connector to delete Datadog organization API keys. Off by default: syncing secrets does not grant deletion.", + "boolField": {} + }, { "name": "sync-schedules", "displayName": "Sync schedules", diff --git a/docs/connector.mdx b/docs/connector.mdx index 196eb233..e176eaf7 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -18,10 +18,10 @@ sidebarTitle: "Datadog" | Roles | | | | | | Teams | | | | | | Schedules | * | | | | -| Secrets - Organization API keys | | | | | +| Secrets - Organization API keys | | | ‡ | ‡ | | Secrets - Service account application keys | | | | † | -[This connector can sync secrets](/product/admin/inventory) and display them on the **Inventory** page. Organization API keys and service account application keys are synced and shown as distinct secret kinds; only application keys owned by a Datadog service account can be issued. +[This connector can sync secrets](/product/admin/inventory) and display them on the **Inventory** page. Organization API keys and service account application keys are synced, issued, and shown as distinct secret kinds: they are two different kinds of API key, not two spellings of one, and a request names which kind it wants. Service account application keys are the default kind. An application key can be issued and revoked through C1 when **Sync secrets** is enabled, provided the selected Datadog user is a service account. Datadog does not support an expiration date when creating an application key. @@ -31,6 +31,8 @@ Credential issuance targets a Datadog service account only. C1 re-checks at issu Revoking a service account application key requires the caller to supply that owning service account alongside the key itself. Until the requesting workflow threads it through, a revoke request that omits it fails rather than guessing which service account owns the key. +‡Organization API key issuance and revocation both require **Allow organization API key deletion**, which is off by default and is separate from **Sync secrets**. An organization API key belongs to the whole organization rather than to the person it was issued to, and it cannot be scoped, so C1 will not mint one it has no permission to revoke. With the setting off, organization API keys still sync; they simply cannot be issued or deleted. + *Schedules and application-key issuance are not enabled by default. Enable **Sync schedules** or **Sync secrets**, respectively, when configuring the connector. †Revoking a service account application key requires the request to name the owning service account as well as the key, because Datadog has no delete-by-key-id-alone form for these keys. A revoke request that omits it is refused rather than guessing. Until the requesting workflow supplies it, revocation is implemented and advertised but will not complete — see the note above. @@ -52,7 +54,7 @@ Configuring the connector requires you to pass in credentials generated in Datad A user with the **Connector Administrator** or **Super Administrator** role in C1 and the **Datadog Admin** or **Datadog standard** role in Datadog must perform this task. -If your user has a custom Datadog role, make sure it includes **User App Keys**, **User Access Invite**, and **User Access Manage** to create, update, enable, and disable users from C1. If you enable **Sync secrets**, also add **Service Account Write**, which governs syncing, issuing, and revoking service account application keys, plus **API Keys Read** and **API Keys Delete** to sync and revoke organization API keys. **Service Account Write** is required, not optional: with **Sync secrets** enabled, a role that lacks it fails the sync rather than syncing an application-key inventory that is silently missing keys. +If your user has a custom Datadog role, make sure it includes **User App Keys**, **User Access Invite**, and **User Access Manage** to create, update, enable, and disable users from C1. If you enable **Sync secrets**, also add **Service Account Write**, which governs syncing, issuing, and revoking service account application keys, plus **API Keys Read** to sync organization API keys. Add **API Keys Write** and **API Keys Delete** only if you also enable **Allow organization API key deletion**; without that setting the connector never creates or deletes an organization API key, so those two permissions are not needed. **Service Account Write** is required, not optional: with **Sync secrets** enabled, a role that lacks it fails the sync rather than syncing an application-key inventory that is silently missing keys. ### Locate your Datadog site @@ -152,6 +154,9 @@ To complete this task, you'll need: **Optional.** Enable **Sync secrets** to display them on the [Inventory page](/product/admin/inventory). + **Optional.** Enable **Allow organization API key deletion** to let C1 issue and revoke Datadog organization API keys. Leave it off unless you want C1 to be able to delete organization-wide keys; enabling **Sync secrets** alone does not grant this. + + **Optional.** Enable **Sync schedules**. @@ -241,6 +246,10 @@ stringData: # Optional: include if you want to sync secrets (API keys) from Datadog BATON_SYNC_SECRETS: true + + # Optional: include ONLY if you want C1 to issue and delete organization-wide + # Datadog API keys. Syncing secrets does not grant this on its own. + BATON_ALLOW_ORG_API_KEY_DELETION: true ``` See the connector's README or run `--help` to see all available configuration flags and environment variables. From 8d6d0b8a0c4f68300ad763c9e0b593ac1bf83fe6 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 28 Aug 2026 00:08:51 +0000 Subject: [PATCH 39/49] test(secrets): add a live smoke test for organization API key issuance TestCredentialIssueLifecycle covers the service account application key arm only. The organization API key arm is the new one and had no live coverage, so a caller could not tell from the suite whether the dispatch actually reaches a different Datadog API. The new test mirrors the existing opt-in guard and always revokes what it mints. It asserts the issued resource comes back as the kind that was requested rather than the preferred one, that it carries no parent resource id, and that the key authenticates before revocation and stops afterwards. The probe is GET /api/v1/validate, which authenticates on the API key alone. An organization API key has no application key to pair with, so the connector's own ValidateCredentials path would not isolate the credential under test. Co-authored-by: c1-squire-dev[bot] --- .../credential_org_key_smoke_test.go | 166 ++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100644 pkg/connector/credential_org_key_smoke_test.go diff --git a/pkg/connector/credential_org_key_smoke_test.go b/pkg/connector/credential_org_key_smoke_test.go new file mode 100644 index 00000000..ab5e7d22 --- /dev/null +++ b/pkg/connector/credential_org_key_smoke_test.go @@ -0,0 +1,166 @@ +package connector + +import ( + "context" + "fmt" + "net/http" + "os" + "testing" + "time" + + cfg "github.com/conductorone/baton-datadog/pkg/config" + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" +) + +// TestOrganizationAPIKeyIssueLifecycle is the opt-in live-provider smoke test +// for the second issuance kind. It mints a real Datadog ORGANIZATION API key, +// which is org-wide and unscoped, and always attempts to revoke it before +// returning. Run it only in a disposable Datadog organization: +// +// DATADOG_CREDENTIAL_SMOKE=1 DATADOG_SMOKE_SITE=datadoghq.com \ +// DATADOG_SMOKE_API_KEY=... DATADOG_SMOKE_APP_KEY=... \ +// DATADOG_SMOKE_SERVICE_ACCOUNT_ID= \ +// go test ./pkg/connector -run TestOrganizationAPIKeyIssueLifecycle -count=1 +// +// The service account id is only the identity the key is recorded as vended +// to. Unlike an application key, an organization API key has no provider-side +// owner, so Datadog never associates the key with that user. +func TestOrganizationAPIKeyIssueLifecycle(t *testing.T) { + if os.Getenv("DATADOG_CREDENTIAL_SMOKE") != "1" { + t.Skip("set DATADOG_CREDENTIAL_SMOKE=1 to run against Datadog") + } + + site := os.Getenv("DATADOG_SMOKE_SITE") + apiKey := os.Getenv("DATADOG_SMOKE_API_KEY") + appKey := os.Getenv("DATADOG_SMOKE_APP_KEY") + identityID := os.Getenv("DATADOG_SMOKE_SERVICE_ACCOUNT_ID") + require.NotEmpty(t, site, "DATADOG_SMOKE_SITE is required") + require.NotEmpty(t, apiKey, "DATADOG_SMOKE_API_KEY is required") + require.NotEmpty(t, appKey, "DATADOG_SMOKE_APP_KEY is required") + require.NotEmpty(t, identityID, "DATADOG_SMOKE_SERVICE_ACCOUNT_ID is required as the identity the key is vended to") + + ctx := context.Background() + builder, _, err := New(ctx, &cfg.Datadog{ + Site: site, + ApiKey: apiKey, + AppKey: appKey, + SyncSecrets: true, + AllowOrgApiKeyDeletion: true, + }, nil) + require.NoError(t, err) + datadogConnector, ok := builder.(*Datadog) + require.True(t, ok) + + // The grant is what puts the organization API key on the menu at all. + details, _, err := newCredentialUserBuilder(datadogConnector.wrapper, datadogConnector.AllowOrgAPIKeyDeletion).IssueCapabilityDetails(ctx) + require.NoError(t, err) + var advertised bool + for _, descriptor := range details.GetOptions() { + if descriptor.GetSecretResourceTypeId() == apiTokenResourceType.Id { + advertised = true + } + } + require.True(t, advertised, "organization API key issuance must be advertised when the grant is set") + + issuer := newCredentialUserBuilder(datadogConnector.wrapper, true) + requestID := "orgsmoke-" + time.Now().UTC().Format("20060102T150405") + t.Logf("issuing Datadog organization API key with request id %q", requestID) + issued, err := issuer.Issue(ctx, &connectorbuilder.CredentialIssueInput{ + IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: identityID}, + RequestID: requestID, + CredentialOptions: v2.CredentialIssueOptions_builder{ + SecretResourceTypeId: apiTokenResourceType.Id, + ApiKey: v2.CredentialIssueOptions_ApiKey_builder{}.Build(), + }.Build(), + }) + require.NoError(t, err) + revoked := false + t.Cleanup(func() { + if revoked || issued == nil || issued.Secret == nil || issued.Secret.GetId().GetResource() == "" { + return + } + _, deleteErr := newDeletableAPITokenBuilder(datadogConnector.wrapper).Delete(ctx, issued.Secret.GetId(), nil) + if status.Code(deleteErr) != codes.NotFound { + require.NoError(t, deleteErr, "Datadog organization API key cleanup failed: %s", issued.Secret.GetId().GetResource()) + } + }) + + require.NotNil(t, issued.Secret) + require.Equal(t, apiTokenResourceType.Id, issued.Secret.GetId().GetResourceType(), + "the issued resource must come back as the kind that was requested, not the preferred one") + require.Nil(t, issued.Secret.GetParentResourceId(), + "an organization API key belongs to the organization, not to the identity it was vended to") + require.Len(t, issued.PlaintextData, 1) + require.Equal(t, "api_key", issued.PlaintextData[0].GetName()) + require.NotEmpty(t, issued.PlaintextData[0].GetBytes()) + + orgKeyID := issued.Secret.GetId().GetResource() + t.Logf("issued organization API key id=%s; plaintext material returned but not logged", maskedValue(orgKeyID)) + + found, err := datadogConnector.wrapper.FindAPIKeyByName(ctx, "c1-"+requestID) + require.NoError(t, err) + require.NotNil(t, found, "issued organization API key id=%s not found via ListAPIKeys", maskedValue(orgKeyID)) + t.Logf("confirmed organization API key id=%s exists in Datadog", maskedValue(orgKeyID)) + + issuedKey := string(issued.PlaintextData[0].GetBytes()) + t.Logf("waiting for issued organization API key id=%s to authenticate", maskedValue(orgKeyID)) + require.Eventually(t, func() bool { + ok, err := orgAPIKeyValidates(ctx, site, issuedKey) + if err != nil { + t.Logf("issued organization API key not usable yet: %v", err) + return false + } + return ok + }, 30*time.Second, time.Second, "issued organization API key did not become usable") + t.Logf("confirmed issued organization API key id=%s authenticates with Datadog", maskedValue(orgKeyID)) + + t.Logf("revoking organization API key id=%s", maskedValue(orgKeyID)) + _, err = newDeletableAPITokenBuilder(datadogConnector.wrapper).Delete(ctx, issued.Secret.GetId(), nil) + require.NoError(t, err, "revoke issued Datadog organization API key") + t.Logf("waiting for revoked organization API key id=%s to stop authenticating", maskedValue(orgKeyID)) + require.Eventually(t, func() bool { + ok, err := orgAPIKeyValidates(ctx, site, issuedKey) + if err != nil { + t.Logf("revocation probe failed without an answer; retrying: %v", err) + return false + } + return !ok + }, 30*time.Second, time.Second, "revoked organization API key still authenticates with Datadog") + t.Logf("confirmed revoked organization API key id=%s no longer authenticates", maskedValue(orgKeyID)) + revoked = true + + found, err = datadogConnector.wrapper.FindAPIKeyByName(ctx, "c1-"+requestID) + require.NoError(t, err) + require.Nil(t, found, "revoked organization API key id=%s is still listed", maskedValue(orgKeyID)) + t.Logf("confirmed organization API key id=%s is no longer listed", maskedValue(orgKeyID)) +} + +// orgAPIKeyValidates asks Datadog whether one organization API key is live. +// GET /api/v1/validate authenticates on the API key alone, which is what makes +// it the right probe here: an organization API key has no application key to +// pair with, so the connector's ValidateCredentials path would not isolate the +// credential under test. +func orgAPIKeyValidates(ctx context.Context, site, apiKey string) (bool, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api."+site+"/api/v1/validate", nil) + if err != nil { + return false, err + } + req.Header.Set("DD-API-KEY", apiKey) + resp, err := http.DefaultClient.Do(req) + if err != nil { + return false, err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK: + return true, nil + case http.StatusForbidden, http.StatusUnauthorized: + return false, nil + default: + return false, fmt.Errorf("unexpected status %d from /api/v1/validate", resp.StatusCode) + } +} From e11cbf50128ce8c5b17c1a8e9828e3fb4914d816 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 28 Aug 2026 03:08:53 +0000 Subject: [PATCH 40/49] fix(secrets): do not fail the sync on a disabled service account MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Datadog answers 404, not an empty list, when asked for a disabled service account's application keys. listApplicationKeyPage treats 404 as fatal, so one disabled service account failed the entire application-key walk — and Datadog never deletes users, it only disables them. Any organization that has ever disabled a service account could therefore never sync a single application key, and the resource type never appeared in C1 at all. The walk now skips service accounts Datadog reports as disabled. They cannot authenticate, so no live credential is dropped. A 404 on an account this walk did choose to visit still fails closed: that one was enabled when the users page was read, so not being readable now is a real anomaly. Found against a live Datadog organization, where it kept the service-account application key from ever reaching C1. Co-authored-by: c1-squire-dev[bot] --- pkg/connector/application_key.go | 19 +++++- pkg/connector/credential_lifecycle_test.go | 76 ++++++++++++++++++++++ 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/pkg/connector/application_key.go b/pkg/connector/application_key.go index 6018aedf..5c9c40c5 100644 --- a/pkg/connector/application_key.go +++ b/pkg/connector/application_key.go @@ -172,6 +172,19 @@ func (o *applicationKeyBuilder) listServiceAccountsPage( if serviceAccountID == "" { continue } + // Datadog answers 404, not an empty list, for a disabled service + // account's application keys, and listApplicationKeyPage fails closed + // on 404. Datadog never deletes users, only disables them, so visiting + // one would make every application key in the organization permanently + // unsyncable. A disabled account cannot authenticate, so skipping it + // drops no live credential. + if user.Attributes.GetDisabled() { + ctxzap.Extract(ctx).Debug( + "baton-datadog: skipping application keys for a disabled service account", + zap.String("service_account_id", serviceAccountID), + ) + continue + } bag.Push(pagination.PageState{ ResourceTypeID: userResourceType.Id, ResourceID: serviceAccountID, @@ -217,9 +230,13 @@ func (o *applicationKeyBuilder) listApplicationKeyPage( serviceAccountID, err) } if code == codes.NotFound { + // Disabled service accounts are filtered out before they reach here + // (see listServiceAccountsPage), so a 404 on one this walk chose to + // visit means it was enabled when the users page was read and is + // not readable now. return nil, nil, fmt.Errorf( "baton-datadog: list application keys for service account %q: %w "+ - "(the service account was not found, and may have been deleted mid-sync)", + "(the service account was not found; it may have been deleted or disabled mid-sync)", serviceAccountID, err) } return nil, nil, fmt.Errorf("baton-datadog: list application keys for service account %q: %w", serviceAccountID, err) diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index f14a69cf..abc24daf 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -1244,3 +1244,79 @@ func TestApiTokenListSurvivesMalformedTimestamps(t *testing.T) { // Every key carries two unparseable fields, and each is reported. require.Equal(t, keys*2, warned, "every unparseable timestamp must be reported") } + +// TestApplicationKeyBuilderSkipsDisabledServiceAccounts is the regression for a +// sync-wide outage: Datadog answers 404, not an empty list, for a disabled +// service account's application keys, and listApplicationKeyPage fails closed on +// 404. Because Datadog never deletes users -- only disables them -- one disabled +// service account made every application key in the organization permanently +// unsyncable. The walk must not visit them at all. +func TestApplicationKeyBuilderSkipsDisabledServiceAccounts(t *testing.T) { + t.Parallel() + + const enabledID = "sa-enabled" + const disabledID = "sa-disabled" + usersPage := `[` + + `{"id":"` + disabledID + `","attributes":{"service_account":true,"disabled":true,"status":"Disabled"}},` + + `{"id":"` + enabledID + `","attributes":{"service_account":true,"disabled":false,"status":"Active"}}` + + `]` + + var mu sync.Mutex + visited := map[string]int{} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + page := 0 + if raw := r.URL.Query().Get("page[number]"); raw != "" { + _, _ = fmt.Sscanf(raw, "%d", &page) + } + if r.URL.Path == "/api/v2/users" { + if page == 0 { + _, _ = w.Write([]byte(`{"data":` + usersPage + `}`)) + } else { + _, _ = w.Write([]byte(`{"data":[]}`)) + } + return + } + const prefix = "/api/v2/service_accounts/" + const suffix = "/application_keys" + if strings.HasPrefix(r.URL.Path, prefix) && strings.HasSuffix(r.URL.Path, suffix) { + id := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, prefix), suffix) + mu.Lock() + visited[id]++ + mu.Unlock() + if id == disabledID { + // What Datadog actually does for a disabled service account. + w.WriteHeader(http.StatusNotFound) + _, _ = w.Write([]byte(`{"errors":["Not Found"]}`)) + return + } + if page == 0 { + _, _ = w.Write([]byte(`{"data":` + appKeyPageJSON("live", 1) + `}`)) + } else { + _, _ = w.Write([]byte(`{"data":[]}`)) + } + return + } + w.WriteHeader(http.StatusInternalServerError) + })) + defer server.Close() + + builder := newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)) + var synced []*v2.Resource + token := "" + for call := 0; call < 50; call++ { + got, results, err := builder.List(context.Background(), nil, rs.SyncOpAttrs{PageToken: pagination.Token{Token: token}}) + require.NoError(t, err, "a disabled service account must not fail the walk") + synced = append(synced, got...) + if results == nil || results.NextPageToken == "" { + break + } + token = results.NextPageToken + } + + mu.Lock() + defer mu.Unlock() + require.Zero(t, visited[disabledID], "the disabled service account must never be requested") + require.Positive(t, visited[enabledID], "the enabled service account must still be walked") + require.Len(t, synced, 1, "the enabled service account's application key must still sync") +} From 057ddae54153230545ba96f19c2882410a3d05df Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 28 Aug 2026 03:40:51 +0000 Subject: [PATCH 41/49] test(secrets): probe the issued org key through the connector's own client MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The organization API key smoke test hand-rolled an http.Client call to /api/v1/validate. gosec's taint analysis flagged it (G704), and it was the wrong shape anyway: every other probe in this package goes through client.DatadogClient. ValidateCredentials calls the same endpoint. It authenticates on the API key alone, which is what makes it the right probe for an organization key — that kind has no application key to pair with, so a probe requiring one would not isolate the credential under test. The empty application key is deliberate and now documented. Datadog refusing the credential is now returned as "not valid" rather than as a probe error, which is what the revocation check was already treating it as. Co-authored-by: c1-squire-dev[bot] --- .../credential_org_key_smoke_test.go | 34 ++++++++----------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/pkg/connector/credential_org_key_smoke_test.go b/pkg/connector/credential_org_key_smoke_test.go index ab5e7d22..f32b4847 100644 --- a/pkg/connector/credential_org_key_smoke_test.go +++ b/pkg/connector/credential_org_key_smoke_test.go @@ -2,12 +2,12 @@ package connector import ( "context" - "fmt" - "net/http" "os" "testing" "time" + "github.com/DataDog/datadog-api-client-go/v2/api/datadog" + "github.com/conductorone/baton-datadog/pkg/client" cfg "github.com/conductorone/baton-datadog/pkg/config" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" @@ -140,27 +140,21 @@ func TestOrganizationAPIKeyIssueLifecycle(t *testing.T) { } // orgAPIKeyValidates asks Datadog whether one organization API key is live. +// It goes through the connector's own client rather than a hand-rolled request: // GET /api/v1/validate authenticates on the API key alone, which is what makes -// it the right probe here: an organization API key has no application key to -// pair with, so the connector's ValidateCredentials path would not isolate the -// credential under test. +// it the right probe here. An organization API key has no application key to +// pair with, so a probe that required one would not isolate the credential +// under test. The empty application key is deliberate. func orgAPIKeyValidates(ctx context.Context, site, apiKey string) (bool, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api."+site+"/api/v1/validate", nil) - if err != nil { - return false, err - } - req.Header.Set("DD-API-KEY", apiKey) - resp, err := http.DefaultClient.Do(req) + cfg := datadog.NewConfiguration() + probe := client.NewDatadogClient(nil, datadog.NewAPIClient(cfg), site, apiKey, "") + resp, err := probe.ValidateCredentials(ctx) if err != nil { + // Datadog refusing the credential is the answer, not a probe failure. + if isCredentialRejection(err) { + return false, nil + } return false, err } - defer resp.Body.Close() - switch resp.StatusCode { - case http.StatusOK: - return true, nil - case http.StatusForbidden, http.StatusUnauthorized: - return false, nil - default: - return false, fmt.Errorf("unexpected status %d from /api/v1/validate", resp.StatusCode) - } + return resp.GetValid(), nil } From 1100f0acf0f342d6da8892d10fc93dd762661d3f Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 28 Aug 2026 03:45:27 +0000 Subject: [PATCH 42/49] test(secrets): drop credential-shaped literals from the gate test gosec flags G101 on a struct literal that assigns string constants to fields named apiKey and appKey, even in a test against a fake provider. These tests read advertised capabilities, which never touch the connector's own credentials, so the fields were doing nothing. Leaving them unset removes the finding rather than suppressing it. Co-authored-by: c1-squire-dev[bot] --- pkg/connector/credential_gate_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/connector/credential_gate_test.go b/pkg/connector/credential_gate_test.go index 8178e646..78672a72 100644 --- a/pkg/connector/credential_gate_test.go +++ b/pkg/connector/credential_gate_test.go @@ -13,11 +13,12 @@ import ( // fake provider, so a test can read the capabilities C1 would actually be // advertised for a given flag combination. func newGateTestConnector(serverURL string, syncSecrets, allowOrgAPIKeyDeletion bool) *Datadog { + // Only the wrapper and the flags matter here: these tests read advertised + // capabilities, which never touch the connector's own credentials. Leaving + // them unset keeps credential-shaped literals out of the package. return &Datadog{ wrapper: newLifecycleTestWrapper(serverURL), site: "example.com", - apiKey: "connector-api-key", - appKey: "connector-app-key", SyncSecrets: syncSecrets, AllowOrgAPIKeyDeletion: allowOrgAPIKeyDeletion, } From 2cba89531725e223d180e8f6f5c7954bd280820d Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:03:51 +0000 Subject: [PATCH 43/49] chore(deps): pin baton-sdk v0.26.0 The credential type discriminator is in a released tag now, so the pre-release pseudo-version this pinned can go. It pointed at the pull request head, which the squash merge has since made unreachable. The vendored diff is smaller than the version jump suggests: the discriminator's generated sources are byte-identical, because the pseudo-version already carried them. What changes is everything else that landed on main while the pull request was open. Co-authored-by: c1-squire-dev[bot] --- go.mod | 2 +- go.sum | 4 ++-- .../pb/c1/connector/v2/annotation_trait.pb.go | 14 ++++++++++---- .../v2/annotation_trait_protoopaque.pb.go | 14 ++++++++++---- .../pb/c1/connector/v2/resource.pb.go | 12 +++++++++--- .../c1/connector/v2/resource_protoopaque.pb.go | 12 +++++++++--- .../baton-sdk/pb/c1/storage/v3/records.pb.go | 10 +++++++--- .../pb/c1/storage/v3/records_protoopaque.pb.go | 10 +++++++--- .../baton-sdk/pkg/field/decode_hooks.go | 18 ++++++++++++++++-- .../baton-sdk/pkg/types/resource/resource.go | 5 +++-- .../pkg/types/resource/resource_attrs.go | 5 +++-- vendor/modules.txt | 2 +- 12 files changed, 78 insertions(+), 30 deletions(-) diff --git a/go.mod b/go.mod index 5db1fa0d..82003de5 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.25.2 require ( github.com/DataDog/datadog-api-client-go/v2 v2.43.0 - github.com/conductorone/baton-sdk v0.25.2-0.20260827221151-1ff7ce2d6fda + github.com/conductorone/baton-sdk v0.26.0 github.com/ennyjfrick/ruleguard-logfatal v0.0.2 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/quasilyte/go-ruleguard/dsl v0.3.23 diff --git a/go.sum b/go.sum index 458a7ea2..b3f3a2fa 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b h1:VXvSNzmr8hMj8 github.com/cockroachdb/swiss v0.0.0-20251224182025-b0f6560f979b/go.mod h1:yBRu/cnL4ks9bgy4vAASdjIW+/xMlFwuHKqtmh3GZQg= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= -github.com/conductorone/baton-sdk v0.25.2-0.20260827221151-1ff7ce2d6fda h1:vap+5POBHWfabs1h60lP3rXv9Mt209oE7kw857FxyZM= -github.com/conductorone/baton-sdk v0.25.2-0.20260827221151-1ff7ce2d6fda/go.mod h1:SKm95z4KkQ23Tufo2ys88lVzbwKb0AQEbKee5GE0Lig= +github.com/conductorone/baton-sdk v0.26.0 h1:aNKg81BhPAVGyYe+W4czZJnL9hzJEUkDinbt0klOeo4= +github.com/conductorone/baton-sdk v0.26.0/go.mod h1:SKm95z4KkQ23Tufo2ys88lVzbwKb0AQEbKee5GE0Lig= github.com/conductorone/dpop v0.2.6 h1:fakwai/Xm2b/fcDUwJN41WtcSI/2UhQOyRIVvnnrrNA= github.com/conductorone/dpop v0.2.6/go.mod h1:gyo8TtzB9SCFCsjsICH4IaLZ7y64CcrDXMOPBwfq/3s= github.com/conductorone/dpop/integrations/dpop_grpc v0.2.4 h1:lYxYi9/WTSL9sE96CO0QF2BY3kehs8dTTApI134TGCA= diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_trait.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_trait.pb.go index 129f9e71..bd2302aa 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_trait.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_trait.pb.go @@ -80,6 +80,9 @@ const ( UserTrait_Status_STATUS_ENABLED UserTrait_Status_Status = 1 UserTrait_Status_STATUS_DISABLED UserTrait_Status_Status = 2 UserTrait_Status_STATUS_DELETED UserTrait_Status_Status = 3 + // Account creation was initiated but the account is not yet usable, such + // as an invitation that has not been accepted. + UserTrait_Status_STATUS_PENDING UserTrait_Status_Status = 4 ) // Enum value maps for UserTrait_Status_Status. @@ -89,12 +92,14 @@ var ( 1: "STATUS_ENABLED", 2: "STATUS_DISABLED", 3: "STATUS_DELETED", + 4: "STATUS_PENDING", } UserTrait_Status_Status_value = map[string]int32{ "STATUS_UNSPECIFIED": 0, "STATUS_ENABLED": 1, "STATUS_DISABLED": 2, "STATUS_DELETED": 3, + "STATUS_PENDING": 4, } ) @@ -2957,7 +2962,7 @@ var File_c1_connector_v2_annotation_trait_proto protoreflect.FileDescriptor const file_c1_connector_v2_annotation_trait_proto_rawDesc = "" + "\n" + - "&c1/connector/v2/annotation_trait.proto\x12\x0fc1.connector.v2\x1a\x1bc1/connector/v2/asset.proto\x1a\x1ec1/connector/v2/resource.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x17validate/validate.proto\"\x9d\v\n" + + "&c1/connector/v2/annotation_trait.proto\x12\x0fc1.connector.v2\x1a\x1bc1/connector/v2/asset.proto\x1a\x1ec1/connector/v2/resource.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x17validate/validate.proto\"\xb1\v\n" + "\tUserTrait\x128\n" + "\x06emails\x18\x01 \x03(\v2 .c1.connector.v2.UserTrait.EmailR\x06emails\x12=\n" + "\x06status\x18\x02 \x01(\v2!.c1.connector.v2.UserTrait.StatusB\x02\x18\x01R\x06status\x125\n" + @@ -2980,16 +2985,17 @@ const file_c1_connector_v2_annotation_trait_proto_rawDesc = "" + "\x05Email\x12!\n" + "\aaddress\x18\x01 \x01(\tB\a\xfaB\x04r\x02`\x01R\aaddress\x12\x1d\n" + "\n" + - "is_primary\x18\x02 \x01(\bR\tisPrimary\x1a\xdc\x01\n" + + "is_primary\x18\x02 \x01(\bR\tisPrimary\x1a\xf0\x01\n" + "\x06Status\x12J\n" + "\x06status\x18\x01 \x01(\x0e2(.c1.connector.v2.UserTrait.Status.StatusB\b\xfaB\x05\x82\x01\x02\x10\x01R\x06status\x12'\n" + "\adetails\x18\x02 \x01(\tB\r\xfaB\n" + - "r\b \x01(\x80\b\xd0\x01\x01R\adetails\"]\n" + + "r\b \x01(\x80\b\xd0\x01\x01R\adetails\"q\n" + "\x06Status\x12\x16\n" + "\x12STATUS_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSTATUS_ENABLED\x10\x01\x12\x13\n" + "\x0fSTATUS_DISABLED\x10\x02\x12\x12\n" + - "\x0eSTATUS_DELETED\x10\x03\x1a,\n" + + "\x0eSTATUS_DELETED\x10\x03\x12\x12\n" + + "\x0eSTATUS_PENDING\x10\x04\x1a,\n" + "\tMFAStatus\x12\x1f\n" + "\vmfa_enabled\x18\x01 \x01(\bR\n" + "mfaEnabled\x1a,\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_trait_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_trait_protoopaque.pb.go index 2f33747f..cc912e31 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_trait_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/annotation_trait_protoopaque.pb.go @@ -80,6 +80,9 @@ const ( UserTrait_Status_STATUS_ENABLED UserTrait_Status_Status = 1 UserTrait_Status_STATUS_DISABLED UserTrait_Status_Status = 2 UserTrait_Status_STATUS_DELETED UserTrait_Status_Status = 3 + // Account creation was initiated but the account is not yet usable, such + // as an invitation that has not been accepted. + UserTrait_Status_STATUS_PENDING UserTrait_Status_Status = 4 ) // Enum value maps for UserTrait_Status_Status. @@ -89,12 +92,14 @@ var ( 1: "STATUS_ENABLED", 2: "STATUS_DISABLED", 3: "STATUS_DELETED", + 4: "STATUS_PENDING", } UserTrait_Status_Status_value = map[string]int32{ "STATUS_UNSPECIFIED": 0, "STATUS_ENABLED": 1, "STATUS_DISABLED": 2, "STATUS_DELETED": 3, + "STATUS_PENDING": 4, } ) @@ -2902,7 +2907,7 @@ var File_c1_connector_v2_annotation_trait_proto protoreflect.FileDescriptor const file_c1_connector_v2_annotation_trait_proto_rawDesc = "" + "\n" + - "&c1/connector/v2/annotation_trait.proto\x12\x0fc1.connector.v2\x1a\x1bc1/connector/v2/asset.proto\x1a\x1ec1/connector/v2/resource.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x17validate/validate.proto\"\x9d\v\n" + + "&c1/connector/v2/annotation_trait.proto\x12\x0fc1.connector.v2\x1a\x1bc1/connector/v2/asset.proto\x1a\x1ec1/connector/v2/resource.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1egoogle/protobuf/wrappers.proto\x1a\x17validate/validate.proto\"\xb1\v\n" + "\tUserTrait\x128\n" + "\x06emails\x18\x01 \x03(\v2 .c1.connector.v2.UserTrait.EmailR\x06emails\x12=\n" + "\x06status\x18\x02 \x01(\v2!.c1.connector.v2.UserTrait.StatusB\x02\x18\x01R\x06status\x125\n" + @@ -2925,16 +2930,17 @@ const file_c1_connector_v2_annotation_trait_proto_rawDesc = "" + "\x05Email\x12!\n" + "\aaddress\x18\x01 \x01(\tB\a\xfaB\x04r\x02`\x01R\aaddress\x12\x1d\n" + "\n" + - "is_primary\x18\x02 \x01(\bR\tisPrimary\x1a\xdc\x01\n" + + "is_primary\x18\x02 \x01(\bR\tisPrimary\x1a\xf0\x01\n" + "\x06Status\x12J\n" + "\x06status\x18\x01 \x01(\x0e2(.c1.connector.v2.UserTrait.Status.StatusB\b\xfaB\x05\x82\x01\x02\x10\x01R\x06status\x12'\n" + "\adetails\x18\x02 \x01(\tB\r\xfaB\n" + - "r\b \x01(\x80\b\xd0\x01\x01R\adetails\"]\n" + + "r\b \x01(\x80\b\xd0\x01\x01R\adetails\"q\n" + "\x06Status\x12\x16\n" + "\x12STATUS_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSTATUS_ENABLED\x10\x01\x12\x13\n" + "\x0fSTATUS_DISABLED\x10\x02\x12\x12\n" + - "\x0eSTATUS_DELETED\x10\x03\x1a,\n" + + "\x0eSTATUS_DELETED\x10\x03\x12\x12\n" + + "\x0eSTATUS_PENDING\x10\x04\x1a,\n" + "\tMFAStatus\x12\x1f\n" + "\vmfa_enabled\x18\x01 \x01(\bR\n" + "mfaEnabled\x1a,\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource.pb.go index 5cec01e5..b908e96e 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource.pb.go @@ -198,6 +198,9 @@ const ( Status_RESOURCE_STATUS_ENABLED Status_ResourceStatus = 1 Status_RESOURCE_STATUS_DISABLED Status_ResourceStatus = 2 Status_RESOURCE_STATUS_DELETED Status_ResourceStatus = 3 + // Account creation was initiated but the account is not yet usable, such + // as an invitation that has not been accepted. + Status_RESOURCE_STATUS_PENDING Status_ResourceStatus = 4 ) // Enum value maps for Status_ResourceStatus. @@ -207,12 +210,14 @@ var ( 1: "RESOURCE_STATUS_ENABLED", 2: "RESOURCE_STATUS_DISABLED", 3: "RESOURCE_STATUS_DELETED", + 4: "RESOURCE_STATUS_PENDING", } Status_ResourceStatus_value = map[string]int32{ "RESOURCE_STATUS_UNSPECIFIED": 0, "RESOURCE_STATUS_ENABLED": 1, "RESOURCE_STATUS_DISABLED": 2, "RESOURCE_STATUS_DELETED": 3, + "RESOURCE_STATUS_PENDING": 4, } ) @@ -6214,16 +6219,17 @@ const file_c1_connector_v2_resource_proto_rawDesc = "" + "\x0eCreationSource\x12\x1f\n" + "\x1bCREATION_SOURCE_UNSPECIFIED\x10\x00\x12,\n" + "(CREATION_SOURCE_CONNECTOR_LIST_RESOURCES\x10\x01\x127\n" + - "3CREATION_SOURCE_CONNECTOR_LIST_GRANTS_PRINCIPAL_JIT\x10\x02\"\x87\x02\n" + + "3CREATION_SOURCE_CONNECTOR_LIST_GRANTS_PRINCIPAL_JIT\x10\x02\"\xa4\x02\n" + "\x06Status\x12H\n" + "\x06status\x18\x01 \x01(\x0e2&.c1.connector.v2.Status.ResourceStatusB\b\xfaB\x05\x82\x01\x02\x10\x01R\x06status\x12'\n" + "\adetails\x18\x02 \x01(\tB\r\xfaB\n" + - "r\b \x01(\x80\b\xd0\x01\x01R\adetails\"\x89\x01\n" + + "r\b \x01(\x80\b\xd0\x01\x01R\adetails\"\xa6\x01\n" + "\x0eResourceStatus\x12\x1f\n" + "\x1bRESOURCE_STATUS_UNSPECIFIED\x10\x00\x12\x1b\n" + "\x17RESOURCE_STATUS_ENABLED\x10\x01\x12\x1c\n" + "\x18RESOURCE_STATUS_DISABLED\x10\x02\x12\x1b\n" + - "\x17RESOURCE_STATUS_DELETED\x10\x03\"\xb5\x03\n" + + "\x17RESOURCE_STATUS_DELETED\x10\x03\x12\x1b\n" + + "\x17RESOURCE_STATUS_PENDING\x10\x04\"\xb5\x03\n" + "$ResourcesServiceListResourcesRequest\x124\n" + "\x10resource_type_id\x18\x01 \x01(\tB\n" + "\xfaB\ar\x05 \x01(\x80\bR\x0eresourceTypeId\x12S\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource_protoopaque.pb.go index 2e2bf9b7..bddb8755 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/connector/v2/resource_protoopaque.pb.go @@ -198,6 +198,9 @@ const ( Status_RESOURCE_STATUS_ENABLED Status_ResourceStatus = 1 Status_RESOURCE_STATUS_DISABLED Status_ResourceStatus = 2 Status_RESOURCE_STATUS_DELETED Status_ResourceStatus = 3 + // Account creation was initiated but the account is not yet usable, such + // as an invitation that has not been accepted. + Status_RESOURCE_STATUS_PENDING Status_ResourceStatus = 4 ) // Enum value maps for Status_ResourceStatus. @@ -207,12 +210,14 @@ var ( 1: "RESOURCE_STATUS_ENABLED", 2: "RESOURCE_STATUS_DISABLED", 3: "RESOURCE_STATUS_DELETED", + 4: "RESOURCE_STATUS_PENDING", } Status_ResourceStatus_value = map[string]int32{ "RESOURCE_STATUS_UNSPECIFIED": 0, "RESOURCE_STATUS_ENABLED": 1, "RESOURCE_STATUS_DISABLED": 2, "RESOURCE_STATUS_DELETED": 3, + "RESOURCE_STATUS_PENDING": 4, } ) @@ -6138,16 +6143,17 @@ const file_c1_connector_v2_resource_proto_rawDesc = "" + "\x0eCreationSource\x12\x1f\n" + "\x1bCREATION_SOURCE_UNSPECIFIED\x10\x00\x12,\n" + "(CREATION_SOURCE_CONNECTOR_LIST_RESOURCES\x10\x01\x127\n" + - "3CREATION_SOURCE_CONNECTOR_LIST_GRANTS_PRINCIPAL_JIT\x10\x02\"\x87\x02\n" + + "3CREATION_SOURCE_CONNECTOR_LIST_GRANTS_PRINCIPAL_JIT\x10\x02\"\xa4\x02\n" + "\x06Status\x12H\n" + "\x06status\x18\x01 \x01(\x0e2&.c1.connector.v2.Status.ResourceStatusB\b\xfaB\x05\x82\x01\x02\x10\x01R\x06status\x12'\n" + "\adetails\x18\x02 \x01(\tB\r\xfaB\n" + - "r\b \x01(\x80\b\xd0\x01\x01R\adetails\"\x89\x01\n" + + "r\b \x01(\x80\b\xd0\x01\x01R\adetails\"\xa6\x01\n" + "\x0eResourceStatus\x12\x1f\n" + "\x1bRESOURCE_STATUS_UNSPECIFIED\x10\x00\x12\x1b\n" + "\x17RESOURCE_STATUS_ENABLED\x10\x01\x12\x1c\n" + "\x18RESOURCE_STATUS_DISABLED\x10\x02\x12\x1b\n" + - "\x17RESOURCE_STATUS_DELETED\x10\x03\"\xb5\x03\n" + + "\x17RESOURCE_STATUS_DELETED\x10\x03\x12\x1b\n" + + "\x17RESOURCE_STATUS_PENDING\x10\x04\"\xb5\x03\n" + "$ResourcesServiceListResourcesRequest\x124\n" + "\x10resource_type_id\x18\x01 \x01(\tB\n" + "\xfaB\ar\x05 \x01(\x80\bR\x0eresourceTypeId\x12S\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go index 5eb228fb..79c38dea 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records.pb.go @@ -104,6 +104,7 @@ const ( StatusRecord_RESOURCE_STATUS_ENABLED StatusRecord_ResourceStatus = 1 StatusRecord_RESOURCE_STATUS_DISABLED StatusRecord_ResourceStatus = 2 StatusRecord_RESOURCE_STATUS_DELETED StatusRecord_ResourceStatus = 3 + StatusRecord_RESOURCE_STATUS_PENDING StatusRecord_ResourceStatus = 4 ) // Enum value maps for StatusRecord_ResourceStatus. @@ -113,12 +114,14 @@ var ( 1: "RESOURCE_STATUS_ENABLED", 2: "RESOURCE_STATUS_DISABLED", 3: "RESOURCE_STATUS_DELETED", + 4: "RESOURCE_STATUS_PENDING", } StatusRecord_ResourceStatus_value = map[string]int32{ "RESOURCE_STATUS_UNSPECIFIED": 0, "RESOURCE_STATUS_ENABLED": 1, "RESOURCE_STATUS_DISABLED": 2, "RESOURCE_STATUS_DELETED": 3, + "RESOURCE_STATUS_PENDING": 4, } ) @@ -2790,15 +2793,16 @@ var File_c1_storage_v3_records_proto protoreflect.FileDescriptor const file_c1_storage_v3_records_proto_rawDesc = "" + "\n" + - "\x1bc1/storage/v3/records.proto\x12\rc1.storage.v3\x1a\x1bc1/storage/v3/options.proto\x1a\x18c1/storage/v3/refs.proto\x1a\x19google/protobuf/any.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xf8\x01\n" + + "\x1bc1/storage/v3/records.proto\x12\rc1.storage.v3\x1a\x1bc1/storage/v3/options.proto\x1a\x18c1/storage/v3/refs.proto\x1a\x19google/protobuf/any.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x95\x02\n" + "\fStatusRecord\x12B\n" + "\x06status\x18\x01 \x01(\x0e2*.c1.storage.v3.StatusRecord.ResourceStatusR\x06status\x12\x18\n" + - "\adetails\x18\x02 \x01(\tR\adetails\"\x89\x01\n" + + "\adetails\x18\x02 \x01(\tR\adetails\"\xa6\x01\n" + "\x0eResourceStatus\x12\x1f\n" + "\x1bRESOURCE_STATUS_UNSPECIFIED\x10\x00\x12\x1b\n" + "\x17RESOURCE_STATUS_ENABLED\x10\x01\x12\x1c\n" + "\x18RESOURCE_STATUS_DISABLED\x10\x02\x12\x1b\n" + - "\x17RESOURCE_STATUS_DELETED\x10\x03\"\x86\x01\n" + + "\x17RESOURCE_STATUS_DELETED\x10\x03\x12\x1b\n" + + "\x17RESOURCE_STATUS_PENDING\x10\x04\"\x86\x01\n" + "\x15GrantExpandableRecord\x12'\n" + "\x0fentitlement_ids\x18\x01 \x03(\tR\x0eentitlementIds\x12\x18\n" + "\ashallow\x18\x02 \x01(\bR\ashallow\x12*\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go index b276786f..7667fe00 100644 --- a/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go +++ b/vendor/github.com/conductorone/baton-sdk/pb/c1/storage/v3/records_protoopaque.pb.go @@ -104,6 +104,7 @@ const ( StatusRecord_RESOURCE_STATUS_ENABLED StatusRecord_ResourceStatus = 1 StatusRecord_RESOURCE_STATUS_DISABLED StatusRecord_ResourceStatus = 2 StatusRecord_RESOURCE_STATUS_DELETED StatusRecord_ResourceStatus = 3 + StatusRecord_RESOURCE_STATUS_PENDING StatusRecord_ResourceStatus = 4 ) // Enum value maps for StatusRecord_ResourceStatus. @@ -113,12 +114,14 @@ var ( 1: "RESOURCE_STATUS_ENABLED", 2: "RESOURCE_STATUS_DISABLED", 3: "RESOURCE_STATUS_DELETED", + 4: "RESOURCE_STATUS_PENDING", } StatusRecord_ResourceStatus_value = map[string]int32{ "RESOURCE_STATUS_UNSPECIFIED": 0, "RESOURCE_STATUS_ENABLED": 1, "RESOURCE_STATUS_DISABLED": 2, "RESOURCE_STATUS_DELETED": 3, + "RESOURCE_STATUS_PENDING": 4, } ) @@ -2672,15 +2675,16 @@ var File_c1_storage_v3_records_proto protoreflect.FileDescriptor const file_c1_storage_v3_records_proto_rawDesc = "" + "\n" + - "\x1bc1/storage/v3/records.proto\x12\rc1.storage.v3\x1a\x1bc1/storage/v3/options.proto\x1a\x18c1/storage/v3/refs.proto\x1a\x19google/protobuf/any.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xf8\x01\n" + + "\x1bc1/storage/v3/records.proto\x12\rc1.storage.v3\x1a\x1bc1/storage/v3/options.proto\x1a\x18c1/storage/v3/refs.proto\x1a\x19google/protobuf/any.proto\x1a\x1cgoogle/protobuf/struct.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\x95\x02\n" + "\fStatusRecord\x12B\n" + "\x06status\x18\x01 \x01(\x0e2*.c1.storage.v3.StatusRecord.ResourceStatusR\x06status\x12\x18\n" + - "\adetails\x18\x02 \x01(\tR\adetails\"\x89\x01\n" + + "\adetails\x18\x02 \x01(\tR\adetails\"\xa6\x01\n" + "\x0eResourceStatus\x12\x1f\n" + "\x1bRESOURCE_STATUS_UNSPECIFIED\x10\x00\x12\x1b\n" + "\x17RESOURCE_STATUS_ENABLED\x10\x01\x12\x1c\n" + "\x18RESOURCE_STATUS_DISABLED\x10\x02\x12\x1b\n" + - "\x17RESOURCE_STATUS_DELETED\x10\x03\"\x86\x01\n" + + "\x17RESOURCE_STATUS_DELETED\x10\x03\x12\x1b\n" + + "\x17RESOURCE_STATUS_PENDING\x10\x04\"\x86\x01\n" + "\x15GrantExpandableRecord\x12'\n" + "\x0fentitlement_ids\x18\x01 \x03(\tR\x0eentitlementIds\x12\x18\n" + "\ashallow\x18\x02 \x01(\bR\ashallow\x12*\n" + diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/field/decode_hooks.go b/vendor/github.com/conductorone/baton-sdk/pkg/field/decode_hooks.go index 202ebb56..b3201379 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/field/decode_hooks.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/field/decode_hooks.go @@ -2,7 +2,9 @@ package field import ( "encoding/base64" + "errors" "fmt" + "io/fs" "net/url" "os" "reflect" @@ -75,7 +77,7 @@ func getFileContentFromPath(path string) ([]byte, error) { // Check if the file exists fileInfo, err := os.Stat(path) if err != nil { - return nil, fmt.Errorf("cannot access file: %w", err) + return nil, fmt.Errorf("cannot access file: %w", redactPathError(err)) } // Check file size limit (2MB) @@ -87,11 +89,19 @@ func getFileContentFromPath(path string) ([]byte, error) { // Read the file content, err := os.ReadFile(path) if err != nil { - return nil, fmt.Errorf("error reading file: %w", err) + return nil, fmt.Errorf("error reading file: %w", redactPathError(err)) } return content, nil } +func redactPathError(err error) error { + var pathErr *fs.PathError + if errors.As(err, &pathErr) { + return pathErr.Err + } + return err +} + // parseFileContent returns the file upload content from a string field value. func parseFileContent(data string) ([]byte, error) { if data == "" { @@ -118,6 +128,10 @@ func parseFileContent(data string) ([]byte, error) { func parseJSONBase64DataURL(dataURL string) ([]byte, error) { parsedURL, err := url.Parse(dataURL) if err != nil { + var urlErr *url.Error + if errors.As(err, &urlErr) { + return nil, fmt.Errorf("invalid data URL: %w", urlErr.Err) + } return nil, fmt.Errorf("invalid data URL: %w", err) } diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go b/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go index f77b9c3c..afada08b 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource.go @@ -171,8 +171,9 @@ func syncAgentTraitToResource(r *v2.Resource, at *v2.AgentTrait) { r.SetProfile(at.GetProfile()) } if at.GetStatus() != v2.AgentTrait_AGENT_STATUS_UNSPECIFIED && !r.HasStatus() { - // AgentTrait_AgentStatus and Status_ResourceStatus enum values are - // identical (READY maps to ENABLED). + // AgentTrait_AgentStatus is a numeric prefix of Status_ResourceStatus + // (READY maps to ENABLED); new AgentStatus values must not reuse + // ResourceStatus numbers with different meanings. r.SetStatus(v2.Status_builder{ Status: v2.Status_ResourceStatus(at.GetStatus()), }.Build()) diff --git a/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource_attrs.go b/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource_attrs.go index f0639c9e..f819c796 100644 --- a/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource_attrs.go +++ b/vendor/github.com/conductorone/baton-sdk/pkg/types/resource/resource_attrs.go @@ -87,8 +87,9 @@ func GetStatus(r *v2.Resource) *v2.Status { }.Build() } if agt := (&v2.AgentTrait{}); pickTrait(r, agt) && agt.GetStatus() != v2.AgentTrait_AGENT_STATUS_UNSPECIFIED { - // AgentTrait_AgentStatus and Status_ResourceStatus enum values are - // identical (READY maps to ENABLED). + // AgentTrait_AgentStatus is a numeric prefix of Status_ResourceStatus + // (READY maps to ENABLED); new AgentStatus values must not reuse + // ResourceStatus numbers with different meanings. return v2.Status_builder{ Status: v2.Status_ResourceStatus(agt.GetStatus()), }.Build() diff --git a/vendor/modules.txt b/vendor/modules.txt index 85c0e6ec..832742a1 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -278,7 +278,7 @@ github.com/cockroachdb/swiss # github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 ## explicit; go 1.19 github.com/cockroachdb/tokenbucket -# github.com/conductorone/baton-sdk v0.25.2-0.20260827221151-1ff7ce2d6fda +# github.com/conductorone/baton-sdk v0.26.0 ## explicit; go 1.25.2 github.com/conductorone/baton-sdk/internal/connector github.com/conductorone/baton-sdk/pb/c1/c1z/v1 From ed78713e73a2a3e02fd9d1c685eed8665ca6361c Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:26:52 +0000 Subject: [PATCH 44/49] fix(credentials): revoke an application key without its parent id Delete required parentResourceID to name the owning service account, and no C1 caller populates it, so every revoke of an issued service-account application key failed InvalidArgument and left the key live at the provider -- while the connector advertised CAPABILITY_RESOURCE_DELETE for that type. That is the failure mode api_token.go argues against. The owner is recoverable from the key: Datadog carries it as the owned_by relationship on GetApplicationKey. Delete now reads it when the parent is absent and keeps failing closed only when the lookup cannot name an owner, so the fallback widens what can be revoked without letting Delete guess. Also bounds the users level of the application-key walk. The application-key level already had maxApplicationKeyPages; the users level terminates on an empty page rather than a short one, so a provider that ignores page[number] would re-push child states forever. Co-authored-by: c1-squire-dev[bot] --- pkg/client/client.go | 24 ++++++ pkg/connector/application_key.go | 35 ++++++-- pkg/connector/credential_lifecycle_test.go | 95 +++++++++++++++++----- 3 files changed, 126 insertions(+), 28 deletions(-) diff --git a/pkg/client/client.go b/pkg/client/client.go index 29c052f3..bdb52b3a 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -372,6 +372,30 @@ func (w *DatadogClient) DeleteServiceAccountApplicationKey(ctx context.Context, return nil } +// FindApplicationKeyOwner returns the id of the service account that owns an +// application key. Datadog carries it as the key's owned_by relationship, so a +// caller holding only the key id can still reach the service-account-scoped +// endpoints, which take both ids. +func (w *DatadogClient) FindApplicationKeyOwner(ctx context.Context, appKeyID string) (string, error) { + ctx = w.withAuthContext(ctx) + api := datadogV2.NewKeyManagementApi(w.officialClient) + response, httpRes, err := api.GetApplicationKey(ctx, appKeyID) + if httpRes != nil { + defer httpRes.Body.Close() + } + if err != nil { + return "", wrapOfficialClientError("get application key", httpRes, err) + } + if response.Data == nil || response.Data.Relationships == nil || response.Data.Relationships.OwnedBy == nil { + return "", fmt.Errorf("get application key %s: response omitted the owned_by relationship", appKeyID) + } + ownerID := response.Data.Relationships.OwnedBy.Data.Id + if ownerID == "" { + return "", fmt.Errorf("get application key %s: owned_by names no user", appKeyID) + } + return ownerID, nil +} + // Wrapper methods that handle HTTP response body closing automatically // ListRoleUsers lists users for a specific role and automatically handles HTTP response body closing. diff --git a/pkg/connector/application_key.go b/pkg/connector/application_key.go index 5c9c40c5..86daab4a 100644 --- a/pkg/connector/application_key.go +++ b/pkg/connector/application_key.go @@ -68,10 +68,10 @@ func (o *applicationKeyBuilder) Grants(_ context.Context, _ *v2.Resource, _ reso // ResourceDeleterV2Limited.Delete signature itself is unchanged by this // choice -- it already took two *v2.ResourceId parameters; apiTokenBuilder // (organization API keys, which need only one id) simply discards the -// second one. Whatever C1-side caller eventually populates parentResourceID -// for a real delete is a platform-level change this file does not make and -// does not depend on to be correct: until that caller exists, Delete fails -// closed with InvalidArgument rather than guessing. +// second one. No C1 caller populates parentResourceID today, so Delete falls +// back to the key's owned_by relationship and only fails closed when that +// lookup cannot name an owner either -- a delete capability that is advertised +// but refuses every call would leave issued keys live at the provider. func (o *applicationKeyBuilder) Delete(ctx context.Context, resourceID *v2.ResourceId, parentResourceID *v2.ResourceId) (annotations.Annotations, error) { if resourceID == nil { return nil, status.Error(codes.InvalidArgument, "baton-datadog: service account application key id is required") @@ -80,10 +80,21 @@ func (o *applicationKeyBuilder) Delete(ctx context.Context, resourceID *v2.Resou if isMalformedAPIKeyHandle(appKeyID) { return nil, status.Errorf(codes.InvalidArgument, "baton-datadog: service account application key id %q is malformed", appKeyID) } - if parentResourceID == nil || parentResourceID.GetResourceType() != userResourceType.Id { - return nil, status.Error(codes.InvalidArgument, "baton-datadog: the owning service account id is required to delete a service account application key") + serviceAccountID := "" + if parentResourceID != nil && parentResourceID.GetResourceType() == userResourceType.Id { + serviceAccountID = parentResourceID.GetResource() + } + if serviceAccountID == "" { + owner, err := o.wrapper.FindApplicationKeyOwner(ctx, appKeyID) + if err != nil { + if status.Code(err) == codes.NotFound { + return nil, nil + } + return nil, status.Errorf(codes.InvalidArgument, + "baton-datadog: the owning service account for application key %q could not be determined: %v", appKeyID, err) + } + serviceAccountID = owner } - serviceAccountID := parentResourceID.GetResource() if isMalformedAPIKeyHandle(serviceAccountID) { return nil, status.Errorf(codes.InvalidArgument, "baton-datadog: owning service account id %q is malformed", serviceAccountID) } @@ -102,6 +113,11 @@ func (o *applicationKeyBuilder) Delete(ctx context.Context, resourceID *v2.Resou // beyond any real service account's application-key count. const maxApplicationKeyPages = int64(10_000) +// maxUserPages bounds the users level of the same walk. It terminates on an +// empty page rather than a short one, so a provider that ignores page[number] +// and keeps returning full pages would re-push child states forever. +const maxUserPages = int64(10_000) + // List returns at most one provider page per call. The sync walks two levels: // the users pages, to discover which users are service accounts, and then each // service account's own application-key pages, read through the dedicated @@ -144,6 +160,11 @@ func (o *applicationKeyBuilder) listServiceAccountsPage( bag *pagination.Bag, page int64, ) ([]*v2.Resource, *resource.SyncOpResults, error) { + if page >= maxUserPages { + return nil, nil, fmt.Errorf( + "baton-datadog: exceeded %d users pages while syncing service account application keys", + maxUserPages) + } // Datadog's documented default page[size] is 10, so omitting it would run // this walk at a tenth of the page size apiTokenBuilder.List uses -- ten // times the round-trips for the same users, in a walk that already fans out diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index abc24daf..28ae3cc2 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -447,12 +447,10 @@ func TestApplicationKeyBuilderDeleteUsesServiceAccountAPI(t *testing.T) { } // TestApplicationKeyBuilderDeleteRejectsMalformedHandle: nil ResourceId, an -// empty or control-character handle, a missing/wrong-type/malformed -// parentResourceID (the owning service account) must all fail closed before -// any provider request. Datadog's DeleteServiceAccountApplicationKey needs -// both ids; parentResourceID carries the service account id (see -// application_key.go's Delete doc comment for why that parameter, not a -// packed handle string). +// empty or control-character handle, and a control-character service account +// id must all fail closed before any provider request. An absent parent is a +// separate case -- it is resolvable from the key itself, so it is covered by +// TestApplicationKeyBuilderDeleteResolvesOwnerFromTheKey rather than here. func TestApplicationKeyBuilderDeleteRejectsMalformedHandle(t *testing.T) { validParent := &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID} tests := []struct { @@ -475,21 +473,6 @@ func TestApplicationKeyBuilderDeleteRejectsMalformedHandle(t *testing.T) { resourceID: &v2.ResourceId{ResourceType: serviceAccountApplicationKeyResourceType.Id, Resource: "appkey-\n1"}, parentResourceID: validParent, }, - { - name: "nil parentResourceID (owning service account missing)", - resourceID: &v2.ResourceId{ResourceType: serviceAccountApplicationKeyResourceType.Id, Resource: "appkey-1"}, - parentResourceID: nil, - }, - { - name: "wrong-type parentResourceID", - resourceID: &v2.ResourceId{ResourceType: serviceAccountApplicationKeyResourceType.Id, Resource: "appkey-1"}, - parentResourceID: &v2.ResourceId{ResourceType: apiTokenResourceType.Id, Resource: testServiceAccountID}, - }, - { - name: "empty parentResourceID.Resource", - resourceID: &v2.ResourceId{ResourceType: serviceAccountApplicationKeyResourceType.Id, Resource: "appkey-1"}, - parentResourceID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: ""}, - }, { name: "parentResourceID with control character", resourceID: &v2.ResourceId{ResourceType: serviceAccountApplicationKeyResourceType.Id, Resource: "appkey-1"}, @@ -513,6 +496,76 @@ func TestApplicationKeyBuilderDeleteRejectsMalformedHandle(t *testing.T) { } } +// TestApplicationKeyBuilderDeleteResolvesOwnerFromTheKey: no C1 caller +// populates parentResourceID today, so an absent one must not strand an issued +// key at the provider. Datadog carries the owner on the key's owned_by +// relationship; Delete reads it and then issues the same service-account-scoped +// DELETE it would have made had the parent been supplied. +func TestApplicationKeyBuilderDeleteResolvesOwnerFromTheKey(t *testing.T) { + const handle = "appkey-orphaned-1" + tests := []struct { + name string + parentResourceID *v2.ResourceId + }{ + {name: "nil parentResourceID", parentResourceID: nil}, + {name: "wrong-type parentResourceID", parentResourceID: &v2.ResourceId{ResourceType: apiTokenResourceType.Id, Resource: testServiceAccountID}}, + {name: "empty parentResourceID.Resource", parentResourceID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: ""}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var deletePath string + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch { + case r.Method == http.MethodGet && r.URL.Path == "/api/v2/application_keys/"+handle: + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":{"id":"` + handle + `","type":"application_keys",` + + `"relationships":{"owned_by":{"data":{"id":"` + testServiceAccountID + `","type":"users"}}}}}`)) + case r.Method == http.MethodDelete: + deletePath = r.URL.Path + w.WriteHeader(http.StatusNoContent) + default: + t.Errorf("unexpected request %s %s", r.Method, r.URL.Path) + w.WriteHeader(http.StatusInternalServerError) + } + })) + defer server.Close() + + deleter := newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)) + _, err := deleter.Delete( + context.Background(), + &v2.ResourceId{ResourceType: serviceAccountApplicationKeyResourceType.Id, Resource: handle}, + tt.parentResourceID, + ) + require.NoError(t, err) + require.Equal(t, "/api/v2/service_accounts/"+testServiceAccountID+"/application_keys/"+handle, deletePath) + }) + } +} + +// TestApplicationKeyBuilderDeleteFailsWhenOwnerUnknown: the fallback only +// widens what Delete can revoke, it does not make it guess. A key whose +// owned_by names nobody still fails closed, and still without a DELETE. +func TestApplicationKeyBuilderDeleteFailsWhenOwnerUnknown(t *testing.T) { + const handle = "appkey-ownerless-1" + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete { + t.Errorf("provider should not be asked to delete a key with no resolvable owner") + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"data":{"id":"` + handle + `","type":"application_keys"}}`)) + })) + defer server.Close() + + deleter := newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)) + _, err := deleter.Delete( + context.Background(), + &v2.ResourceId{ResourceType: serviceAccountApplicationKeyResourceType.Id, Resource: handle}, + nil, + ) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} + // --- applicationKeyBuilder.List paging ------------------------------------ // newAppKeyListServer fakes the two endpoints applicationKeyBuilder.List From 4bd1ae5d4400c8a1a6553db4ab5e522f2b4f6211 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:27:28 +0000 Subject: [PATCH 45/49] docs: revocation no longer waits on the caller for the owner The warning and footnote both said a revoke that omits the owning service account fails, and that revocation is advertised but cannot complete until the requesting workflow threads the owner through. The connector now reads the owner from the key, so both claims are stale. Co-authored-by: c1-squire-dev[bot] --- docs/connector.mdx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/connector.mdx b/docs/connector.mdx index e176eaf7..240929c2 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -28,14 +28,14 @@ An application key can be issued and revoked through C1 when **Sync secrets** is Credential issuance targets a Datadog service account only. C1 re-checks at issuance time that the selected user is still a service account, and refuses to issue against a human user. The issued application key is owned by, and scoped to, that service account. -Revoking a service account application key requires the caller to supply that owning service account alongside the key itself. Until the requesting workflow threads it through, a revoke request that omits it fails rather than guessing which service account owns the key. +Revoking a service account application key needs the owning service account as well as the key. C1 does not supply it today, so the connector reads it from the key's own owner record instead. A revoke is refused only when that lookup cannot name an owner either. ‡Organization API key issuance and revocation both require **Allow organization API key deletion**, which is off by default and is separate from **Sync secrets**. An organization API key belongs to the whole organization rather than to the person it was issued to, and it cannot be scoped, so C1 will not mint one it has no permission to revoke. With the setting off, organization API keys still sync; they simply cannot be issued or deleted. *Schedules and application-key issuance are not enabled by default. Enable **Sync schedules** or **Sync secrets**, respectively, when configuring the connector. -†Revoking a service account application key requires the request to name the owning service account as well as the key, because Datadog has no delete-by-key-id-alone form for these keys. A revoke request that omits it is refused rather than guessing. Until the requesting workflow supplies it, revocation is implemented and advertised but will not complete — see the note above. +†Revoking a service account application key needs the owning service account as well as the key, because Datadog has no delete-by-key-id-alone form for these keys. When the request omits it, the connector looks the owner up from the key and proceeds; a key whose owner cannot be identified is refused rather than guessed at — see the note above. ### Connector actions From bbff6aba3c1d45df33fd8edeac66ec544e64ff5e Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:40:26 +0000 Subject: [PATCH 46/49] feat(secrets): gate service account application keys on their own grant Listing a service account's application keys needs Datadog's service_account_write, which api_keys_read does not imply, and a 403 there fails the whole sync rather than skipping the account. Registering that syncer under sync-secrets alone therefore broke every existing sync-secrets install on upgrade, on a permission the operator was never asked for. sync-service-account-application-keys is that ask, off by default. It follows the shape allow-org-api-key-deletion already set in this connector: the syncer is not registered without the grant, so the capability is absent from what C1 is advertised rather than advertised and refused. Once granted, the fail-hard behaviour is unchanged -- a credential absent from a completed sync reads as deleted, so skipping would retire live credentials from the inventory. The grant also gates issuance of that kind, because the SDK refuses an issuance descriptor whose secret resource type has no registered deleter. With secrets synced and neither kind granted there is nothing to issue, so CAPABILITY_CREDENTIAL_ISSUE is absent rather than advertised with an empty option list. baton_capabilities.json is unchanged: capability generation forces every optional surface on, so the document still describes the connector's full capability set. Co-authored-by: c1-squire-dev[bot] --- README.md | 1 + cmd/baton-datadog/main.go | 2 + config_schema.json | 6 ++ docs/connector.mdx | 8 +- pkg/config/conf.gen.go | 1 + pkg/config/config.go | 14 +++ pkg/connector/connector.go | 37 ++++++-- pkg/connector/credential_gate_test.go | 88 ++++++++++++++++++- pkg/connector/credential_issue_kinds_test.go | 12 +-- pkg/connector/credential_lifecycle_test.go | 8 +- .../credential_org_key_smoke_test.go | 4 +- pkg/connector/credential_smoke_test.go | 2 +- pkg/connector/users.go | 28 ++++-- 13 files changed, 176 insertions(+), 35 deletions(-) diff --git a/README.md b/README.md index 53af09d8..2289464b 100644 --- a/README.md +++ b/README.md @@ -96,6 +96,7 @@ Flags: --skip-full-sync This must be set to skip a full sync ($BATON_SKIP_FULL_SYNC) --sync-schedules Whether to sync on-call schedules or not ($BATON_SYNC_SCHEDULES) --sync-secrets Whether to sync secrets or not ($BATON_SYNC_SECRETS) + --sync-service-account-application-keys Sync, issue and revoke Datadog service account application keys. Off by default: requires the Datadog service_account_write permission, and a role without it fails the whole sync. ($BATON_SYNC_SERVICE_ACCOUNT_APPLICATION_KEYS) --ticketing This must be set to enable ticketing support ($BATON_TICKETING) -v, --version version for baton-datadog diff --git a/cmd/baton-datadog/main.go b/cmd/baton-datadog/main.go index 946e530a..5e821791 100644 --- a/cmd/baton-datadog/main.go +++ b/cmd/baton-datadog/main.go @@ -28,6 +28,8 @@ func main() { SyncSecrets: true, SyncSchedules: true, AllowOrgAPIKeyDeletion: true, + + SyncServiceAccountApplicationKeys: true, }), ) } diff --git a/config_schema.json b/config_schema.json index 01080e57..8970765c 100644 --- a/config_schema.json +++ b/config_schema.json @@ -147,6 +147,12 @@ "description": "Allow this connector to delete Datadog organization API keys. Off by default: syncing secrets does not grant deletion.", "boolField": {} }, + { + "name": "sync-service-account-application-keys", + "displayName": "Sync service account application keys", + "description": "Sync, issue and revoke Datadog service account application keys. Off by default: requires the Datadog service_account_write permission, and a role without it fails the whole sync.", + "boolField": {} + }, { "name": "sync-schedules", "displayName": "Sync schedules", diff --git a/docs/connector.mdx b/docs/connector.mdx index 240929c2..fa32d2f6 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -23,7 +23,7 @@ sidebarTitle: "Datadog" [This connector can sync secrets](/product/admin/inventory) and display them on the **Inventory** page. Organization API keys and service account application keys are synced, issued, and shown as distinct secret kinds: they are two different kinds of API key, not two spellings of one, and a request names which kind it wants. Service account application keys are the default kind. -An application key can be issued and revoked through C1 when **Sync secrets** is enabled, provided the selected Datadog user is a service account. Datadog does not support an expiration date when creating an application key. +An application key can be issued and revoked through C1 when **Sync secrets** and **Sync service account application keys** are both enabled, provided the selected Datadog user is a service account. Datadog does not support an expiration date when creating an application key. Credential issuance targets a Datadog service account only. C1 re-checks at issuance time that the selected user is still a service account, and refuses to issue against a human user. The issued application key is owned by, and scoped to, that service account. @@ -33,7 +33,7 @@ Revoking a service account application key needs the owning service account as w ‡Organization API key issuance and revocation both require **Allow organization API key deletion**, which is off by default and is separate from **Sync secrets**. An organization API key belongs to the whole organization rather than to the person it was issued to, and it cannot be scoped, so C1 will not mint one it has no permission to revoke. With the setting off, organization API keys still sync; they simply cannot be issued or deleted. -*Schedules and application-key issuance are not enabled by default. Enable **Sync schedules** or **Sync secrets**, respectively, when configuring the connector. +*Schedules and application keys are not enabled by default. Enable **Sync schedules** or **Sync secrets** plus **Sync service account application keys**, respectively, when configuring the connector. †Revoking a service account application key needs the owning service account as well as the key, because Datadog has no delete-by-key-id-alone form for these keys. When the request omits it, the connector looks the owner up from the key and proceeds; a key whose owner cannot be identified is refused rather than guessed at — see the note above. @@ -54,7 +54,7 @@ Configuring the connector requires you to pass in credentials generated in Datad A user with the **Connector Administrator** or **Super Administrator** role in C1 and the **Datadog Admin** or **Datadog standard** role in Datadog must perform this task. -If your user has a custom Datadog role, make sure it includes **User App Keys**, **User Access Invite**, and **User Access Manage** to create, update, enable, and disable users from C1. If you enable **Sync secrets**, also add **Service Account Write**, which governs syncing, issuing, and revoking service account application keys, plus **API Keys Read** to sync organization API keys. Add **API Keys Write** and **API Keys Delete** only if you also enable **Allow organization API key deletion**; without that setting the connector never creates or deletes an organization API key, so those two permissions are not needed. **Service Account Write** is required, not optional: with **Sync secrets** enabled, a role that lacks it fails the sync rather than syncing an application-key inventory that is silently missing keys. +If your user has a custom Datadog role, make sure it includes **User App Keys**, **User Access Invite**, and **User Access Manage** to create, update, enable, and disable users from C1. If you enable **Sync secrets**, add **API Keys Read** to sync organization API keys. Add **Service Account Write**, which governs syncing, issuing, and revoking service account application keys, only if you also enable **Sync service account application keys**. Add **API Keys Write** and **API Keys Delete** only if you also enable **Allow organization API key deletion**; without that setting the connector never creates or deletes an organization API key, so those two permissions are not needed. **Service Account Write** is required, not optional, once that setting is on: a role that lacks it fails the sync rather than syncing an application-key inventory that is silently missing keys. That is why the setting is off by default — an existing install keeps syncing until the operator grants the permission. ### Locate your Datadog site @@ -154,6 +154,8 @@ To complete this task, you'll need: **Optional.** Enable **Sync secrets** to display them on the [Inventory page](/product/admin/inventory). + **Optional.** Enable **Sync service account application keys** to sync, issue, and revoke them. It requires the **Service Account Write** permission, and a role without it fails the whole sync, so it is off by default and **Sync secrets** alone does not turn it on. + **Optional.** Enable **Allow organization API key deletion** to let C1 issue and revoke Datadog organization API keys. Leave it off unless you want C1 to be able to delete organization-wide keys; enabling **Sync secrets** alone does not grant this. diff --git a/pkg/config/conf.gen.go b/pkg/config/conf.gen.go index 41fe884d..b5a6a962 100644 --- a/pkg/config/conf.gen.go +++ b/pkg/config/conf.gen.go @@ -9,6 +9,7 @@ type Datadog struct { AppKey string `mapstructure:"app-key"` SyncSecrets bool `mapstructure:"sync-secrets"` AllowOrgApiKeyDeletion bool `mapstructure:"allow-org-api-key-deletion"` + SyncServiceAccountApplicationKeys bool `mapstructure:"sync-service-account-application-keys"` SyncSchedules bool `mapstructure:"sync-schedules"` BaseUrl string `mapstructure:"base-url"` } diff --git a/pkg/config/config.go b/pkg/config/config.go index 1e883c16..d4a7ada3 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -48,6 +48,19 @@ var ( field.WithDefaultValue(false), field.WithDisplayName("Allow organization API key deletion"), ) + // SyncServiceAccountApplicationKeys carries the Datadog + // service_account_write permission, which api_keys_read does not imply. + // It is off by default because listing a service account's application + // keys fails the whole sync without that permission: an install already + // running with sync-secrets on would start failing every sync merely by + // upgrading the connector. Once granted, the fail-hard behaviour stands -- + // a credential absent from a completed sync reads as deleted. + SyncServiceAccountApplicationKeys = field.BoolField( + "sync-service-account-application-keys", + field.WithDescription("Sync, issue and revoke Datadog service account application keys. Off by default: requires the Datadog service_account_write permission, and a role without it fails the whole sync."), + field.WithDefaultValue(false), + field.WithDisplayName("Sync service account application keys"), + ) SyncSchedules = field.BoolField( "sync-schedules", field.WithDescription("Whether to sync on-call schedules or not"), @@ -76,6 +89,7 @@ var Config = field.NewConfiguration([]field.SchemaField{ AppKey, SyncSecrets, AllowOrgAPIKeyDeletion, + SyncServiceAccountApplicationKeys, SyncSchedules, BaseURL, }, diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index a06ab201..5427ea95 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -36,13 +36,26 @@ type Datadog struct { // from it, and an install already running with sync-secrets on must not // acquire org-wide key deletion merely by upgrading the connector. AllowOrgAPIKeyDeletion bool + // SyncServiceAccountApplicationKeys is the operator's attestation that the + // connector's Datadog role holds service_account_write. Without it, + // listing a service account's application keys 403s and fails the whole + // sync, so registering that syncer unconditionally would break every + // existing sync-secrets install on upgrade. Off by default for that + // reason, not because the capability is optional in itself. + SyncServiceAccountApplicationKeys bool } // ResourceSyncers returns a ResourceSyncer for each resource type that should be synced from the upstream service. func (d *Datadog) ResourceSyncers(ctx context.Context) []connectorbuilder.ResourceSyncerV2 { + offerServiceAccountKey := d.SyncSecrets && d.SyncServiceAccountApplicationKeys + offerOrgAPIKey := d.SyncSecrets && d.AllowOrgAPIKeyDeletion + // A credential issuer with no advertised kind is not an issuer. With + // secrets synced but neither kind granted, the plain user syncer is + // registered and CAPABILITY_CREDENTIAL_ISSUE is absent rather than + // advertised with an empty option list. userSyncer := connectorbuilder.ResourceSyncerV2(newUserBuilder(d.wrapper)) - if d.SyncSecrets { - userSyncer = newCredentialUserBuilder(d.wrapper, d.AllowOrgAPIKeyDeletion) + if offerServiceAccountKey || offerOrgAPIKey { + userSyncer = newCredentialUserBuilder(d.wrapper, offerOrgAPIKey, offerServiceAccountKey) } resourceSyncers := []connectorbuilder.ResourceSyncerV2{ userSyncer, @@ -59,7 +72,10 @@ func (d *Datadog) ResourceSyncers(ctx context.Context) []connectorbuilder.Resour if d.AllowOrgAPIKeyDeletion { apiTokenSyncer = newDeletableAPITokenBuilder(d.wrapper) } - resourceSyncers = append(resourceSyncers, apiTokenSyncer, newApplicationKeyBuilder(d.wrapper)) + resourceSyncers = append(resourceSyncers, apiTokenSyncer) + if d.SyncServiceAccountApplicationKeys { + resourceSyncers = append(resourceSyncers, newApplicationKeyBuilder(d.wrapper)) + } } if d.SyncSchedules { @@ -152,6 +168,7 @@ func New(ctx context.Context, ddc *cfg.Datadog, _ *cli.ConnectorOpts) (connector syncSecrets := ddc.SyncSecrets syncSchedules := ddc.SyncSchedules allowOrgAPIKeyDeletion := ddc.AllowOrgApiKeyDeletion + syncServiceAccountApplicationKeys := ddc.SyncServiceAccountApplicationKeys // Validate input parameters if site == "" { @@ -194,14 +211,16 @@ func New(ctx context.Context, ddc *cfg.Datadog, _ *cli.ConnectorOpts) (connector wrapper := client.NewDatadogClient(restClient, officialClient, site, apiKey, appKey) return &Datadog{ - site: site, - apiKey: apiKey, - appKey: appKey, - baseURL: baseURL, - client: officialClient, - wrapper: wrapper, + site: site, + apiKey: apiKey, + appKey: appKey, + baseURL: baseURL, + client: officialClient, + wrapper: wrapper, SyncSecrets: syncSecrets, SyncSchedules: syncSchedules, AllowOrgAPIKeyDeletion: allowOrgAPIKeyDeletion, + + SyncServiceAccountApplicationKeys: syncServiceAccountApplicationKeys, }, nil, nil } diff --git a/pkg/connector/credential_gate_test.go b/pkg/connector/credential_gate_test.go index 78672a72..b88c158d 100644 --- a/pkg/connector/credential_gate_test.go +++ b/pkg/connector/credential_gate_test.go @@ -13,17 +13,43 @@ import ( // fake provider, so a test can read the capabilities C1 would actually be // advertised for a given flag combination. func newGateTestConnector(serverURL string, syncSecrets, allowOrgAPIKeyDeletion bool) *Datadog { + // Application-key sync is on for the org-key tests so they read the same + // connector shape those tests were written against; the tests that gate it + // build the connector with newAppKeyGateTestConnector instead. + return newAppKeyGateTestConnector(serverURL, syncSecrets, allowOrgAPIKeyDeletion, true) +} + +func newAppKeyGateTestConnector(serverURL string, syncSecrets, allowOrgAPIKeyDeletion, syncServiceAccountApplicationKeys bool) *Datadog { // Only the wrapper and the flags matter here: these tests read advertised // capabilities, which never touch the connector's own credentials. Leaving // them unset keeps credential-shaped literals out of the package. return &Datadog{ - wrapper: newLifecycleTestWrapper(serverURL), - site: "example.com", - SyncSecrets: syncSecrets, - AllowOrgAPIKeyDeletion: allowOrgAPIKeyDeletion, + wrapper: newLifecycleTestWrapper(serverURL), + site: "example.com", + SyncSecrets: syncSecrets, + AllowOrgAPIKeyDeletion: allowOrgAPIKeyDeletion, + SyncServiceAccountApplicationKeys: syncServiceAccountApplicationKeys, } } +// advertisedResourceTypeCapabilities is resourceTypeCapabilities without the +// fatal: a gate whose point is that a resource type is absent needs to ask +// whether it was advertised at all. +func advertisedResourceTypeCapabilities(t *testing.T, d *Datadog, resourceTypeID string) ([]v2.Capability, bool) { + t.Helper() + ctx := context.Background() + server, err := connectorbuilder.NewConnector(ctx, d) + require.NoError(t, err) + md, err := server.GetMetadata(ctx, &v2.ConnectorServiceGetMetadataRequest{}) + require.NoError(t, err) + for _, rtc := range md.GetMetadata().GetCapabilities().GetResourceTypeCapabilities() { + if rtc.GetResourceType().GetId() == resourceTypeID { + return rtc.GetCapabilities(), true + } + } + return nil, false +} + // resourceTypeCapabilities returns the capabilities the SDK advertises for one // resource type id, going through NewConnector/GetMetadata rather than // inspecting the builders directly: the whole point of the gate is what C1 @@ -97,3 +123,57 @@ func TestOrgAPIKeyDeletePermissionFollowsTheGrant(t *testing.T) { }) } } + +// TestApplicationKeySyncRequiresItsOwnGrant is the upgrade regression: listing +// a service account's application keys needs Datadog's service_account_write, +// which api_keys_read does not imply, and a 403 there fails the whole sync. +// An install already running with sync-secrets on must keep syncing after an +// upgrade rather than start failing on a permission it was never asked for. +func TestApplicationKeySyncRequiresItsOwnGrant(t *testing.T) { + d := newAppKeyGateTestConnector("http://127.0.0.1:1", true, false, false) + + _, advertised := advertisedResourceTypeCapabilities(t, d, serviceAccountApplicationKeyResourceType.Id) + require.False(t, advertised, + "sync-secrets alone must not advertise service account application keys") + + orgKeyCaps, advertised := advertisedResourceTypeCapabilities(t, d, apiTokenResourceType.Id) + require.True(t, advertised, "organization API keys must still sync") + require.Contains(t, orgKeyCaps, v2.Capability_CAPABILITY_SYNC) +} + +func TestApplicationKeySyncAdvertisedWithGrant(t *testing.T) { + d := newAppKeyGateTestConnector("http://127.0.0.1:1", true, false, true) + caps, advertised := advertisedResourceTypeCapabilities(t, d, serviceAccountApplicationKeyResourceType.Id) + require.True(t, advertised) + require.Contains(t, caps, v2.Capability_CAPABILITY_SYNC) + require.Contains(t, caps, v2.Capability_CAPABILITY_RESOURCE_DELETE) +} + +// TestCredentialIssueFollowsTheKindGrants: with secrets synced but neither +// kind granted there is nothing to issue, so the capability must be absent +// rather than advertised with an empty option list. Either grant brings it +// back. +func TestCredentialIssueFollowsTheKindGrants(t *testing.T) { + for _, tt := range []struct { + name string + orgKey bool + appKey bool + wantIssue bool + }{ + {name: "neither kind granted", wantIssue: false}, + {name: "application keys only", appKey: true, wantIssue: true}, + {name: "organization keys only", orgKey: true, wantIssue: true}, + {name: "both kinds", orgKey: true, appKey: true, wantIssue: true}, + } { + t.Run(tt.name, func(t *testing.T) { + d := newAppKeyGateTestConnector("http://127.0.0.1:1", true, tt.orgKey, tt.appKey) + caps, advertised := advertisedResourceTypeCapabilities(t, d, userResourceType.Id) + require.True(t, advertised, "users must sync regardless") + if tt.wantIssue { + require.Contains(t, caps, v2.Capability_CAPABILITY_CREDENTIAL_ISSUE) + } else { + require.NotContains(t, caps, v2.Capability_CAPABILITY_CREDENTIAL_ISSUE) + } + }) + } +} diff --git a/pkg/connector/credential_issue_kinds_test.go b/pkg/connector/credential_issue_kinds_test.go index 3dcea77b..0404d6b4 100644 --- a/pkg/connector/credential_issue_kinds_test.go +++ b/pkg/connector/credential_issue_kinds_test.go @@ -20,7 +20,7 @@ import ( // secret_resource_type_id. func TestIssuanceAdvertisesBothCredentialKinds(t *testing.T) { ctx := context.Background() - details, _, err := newCredentialUserBuilder(newLifecycleTestWrapper("http://127.0.0.1:1"), true).IssueCapabilityDetails(ctx) + details, _, err := newCredentialUserBuilder(newLifecycleTestWrapper("http://127.0.0.1:1"), true, true).IssueCapabilityDetails(ctx) require.NoError(t, err) require.Len(t, details.GetOptions(), 2) @@ -45,12 +45,12 @@ func TestIssuanceAdvertisesBothCredentialKinds(t *testing.T) { // would fail connector startup, so the descriptor has to be absent too. func TestIssuanceOmitsOrgAPIKeyWithoutGrant(t *testing.T) { ctx := context.Background() - details, _, err := newCredentialUserBuilder(newLifecycleTestWrapper("http://127.0.0.1:1"), false).IssueCapabilityDetails(ctx) + details, _, err := newCredentialUserBuilder(newLifecycleTestWrapper("http://127.0.0.1:1"), false, true).IssueCapabilityDetails(ctx) require.NoError(t, err) require.Len(t, details.GetOptions(), 1) require.Equal(t, serviceAccountApplicationKeyResourceType.Id, details.GetOptions()[0].GetSecretResourceTypeId()) - out, err := newCredentialUserBuilder(newLifecycleTestWrapper("http://127.0.0.1:1"), false).Issue(ctx, &connectorbuilder.CredentialIssueInput{ + out, err := newCredentialUserBuilder(newLifecycleTestWrapper("http://127.0.0.1:1"), false, true).Issue(ctx, &connectorbuilder.CredentialIssueInput{ IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID}, RequestID: "req-org-denied", CredentialOptions: v2.CredentialIssueOptions_builder{ @@ -89,7 +89,7 @@ func TestIssueDispatchesOnRequestedCredentialKind(t *testing.T) { })) defer server.Close() - issuer := newCredentialUserBuilder(newLifecycleTestWrapper(server.URL), true) + issuer := newCredentialUserBuilder(newLifecycleTestWrapper(server.URL), true, true) out, err := issuer.Issue(ctx, &connectorbuilder.CredentialIssueInput{ IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID}, RequestID: "req-org", @@ -121,7 +121,7 @@ func TestIssueDispatchesOnRequestedCredentialKind(t *testing.T) { // TestIssueRejectsScopesOnOrgAPIKey: the shape allows scopes, this kind does // not, so the arm has to fail closed rather than mint an unscoped key. func TestIssueRejectsScopesOnOrgAPIKey(t *testing.T) { - issuer := newCredentialUserBuilder(newLifecycleTestWrapper("http://127.0.0.1:1"), true) + issuer := newCredentialUserBuilder(newLifecycleTestWrapper("http://127.0.0.1:1"), true, true) out, err := issuer.Issue(context.Background(), &connectorbuilder.CredentialIssueInput{ IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID}, RequestID: "req-org-scoped", @@ -137,7 +137,7 @@ func TestIssueRejectsScopesOnOrgAPIKey(t *testing.T) { // TestIssueRejectsUnknownCredentialKind: an unadvertised secret resource type // is a protocol mismatch, not a cue to fall back to the preferred arm. func TestIssueRejectsUnknownCredentialKind(t *testing.T) { - issuer := newCredentialUserBuilder(newLifecycleTestWrapper("http://127.0.0.1:1"), true) + issuer := newCredentialUserBuilder(newLifecycleTestWrapper("http://127.0.0.1:1"), true, true) for _, secretType := range []string{"", "not-a-datadog-credential"} { out, err := issuer.Issue(context.Background(), &connectorbuilder.CredentialIssueInput{ IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID}, diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index 28ae3cc2..3c865391 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -239,7 +239,7 @@ func newServiceAccountAppKeyServer(t *testing.T, serviceAccountID, handle, secre func issueServiceAccountAppKey(t *testing.T, ctx context.Context, wrapper *client.DatadogClient, serviceAccountID, requestID string) *connectorbuilder.CredentialIssueOutput { t.Helper() - issuer := newCredentialUserBuilder(wrapper, false) + issuer := newCredentialUserBuilder(wrapper, false, true) out, err := issuer.Issue(ctx, &connectorbuilder.CredentialIssueInput{ IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: serviceAccountID}, RequestID: requestID, @@ -291,7 +291,7 @@ func TestIssueRequiresServiceAccount(t *testing.T) { defer server.Close() wrapper := newLifecycleTestWrapper(server.URL) - issuer := newCredentialUserBuilder(wrapper, false) + issuer := newCredentialUserBuilder(wrapper, false, true) out, err := issuer.Issue(context.Background(), &connectorbuilder.CredentialIssueInput{ IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: humanUserID}, RequestID: "req-reject", @@ -336,7 +336,7 @@ func TestIssueRefusesDuplicateRequest(t *testing.T) { defer server.Close() wrapper := newLifecycleTestWrapper(server.URL) - issuer := newCredentialUserBuilder(wrapper, false) + issuer := newCredentialUserBuilder(wrapper, false, true) out, err := issuer.Issue(context.Background(), &connectorbuilder.CredentialIssueInput{ IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID}, RequestID: requestID, @@ -1038,7 +1038,7 @@ func TestIssuePassesScopesToProviderAndProfile(t *testing.T) { })) defer server.Close() - issuer := newCredentialUserBuilder(newLifecycleTestWrapper(server.URL), false) + issuer := newCredentialUserBuilder(newLifecycleTestWrapper(server.URL), false, true) out, err := issuer.Issue(context.Background(), &connectorbuilder.CredentialIssueInput{ IdentityID: &v2.ResourceId{ResourceType: userResourceType.Id, Resource: testServiceAccountID}, RequestID: "req-scoped", diff --git a/pkg/connector/credential_org_key_smoke_test.go b/pkg/connector/credential_org_key_smoke_test.go index f32b4847..f7965f20 100644 --- a/pkg/connector/credential_org_key_smoke_test.go +++ b/pkg/connector/credential_org_key_smoke_test.go @@ -56,7 +56,7 @@ func TestOrganizationAPIKeyIssueLifecycle(t *testing.T) { require.True(t, ok) // The grant is what puts the organization API key on the menu at all. - details, _, err := newCredentialUserBuilder(datadogConnector.wrapper, datadogConnector.AllowOrgAPIKeyDeletion).IssueCapabilityDetails(ctx) + details, _, err := newCredentialUserBuilder(datadogConnector.wrapper, datadogConnector.AllowOrgAPIKeyDeletion, true).IssueCapabilityDetails(ctx) require.NoError(t, err) var advertised bool for _, descriptor := range details.GetOptions() { @@ -66,7 +66,7 @@ func TestOrganizationAPIKeyIssueLifecycle(t *testing.T) { } require.True(t, advertised, "organization API key issuance must be advertised when the grant is set") - issuer := newCredentialUserBuilder(datadogConnector.wrapper, true) + issuer := newCredentialUserBuilder(datadogConnector.wrapper, true, true) requestID := "orgsmoke-" + time.Now().UTC().Format("20060102T150405") t.Logf("issuing Datadog organization API key with request id %q", requestID) issued, err := issuer.Issue(ctx, &connectorbuilder.CredentialIssueInput{ diff --git a/pkg/connector/credential_smoke_test.go b/pkg/connector/credential_smoke_test.go index f3239e4d..c10f0935 100644 --- a/pkg/connector/credential_smoke_test.go +++ b/pkg/connector/credential_smoke_test.go @@ -51,7 +51,7 @@ func TestCredentialIssueLifecycle(t *testing.T) { datadogConnector, ok := builder.(*Datadog) require.True(t, ok) - issuer := newCredentialUserBuilder(datadogConnector.wrapper, false) + issuer := newCredentialUserBuilder(datadogConnector.wrapper, false, true) requestID := "smoke-" + time.Now().UTC().Format("20060102T150405") t.Logf("issuing Datadog service account application key with request id %q", requestID) issued, err := issuer.Issue(ctx, &connectorbuilder.CredentialIssueInput{ diff --git a/pkg/connector/users.go b/pkg/connector/users.go index 755a0bd8..5442c89d 100644 --- a/pkg/connector/users.go +++ b/pkg/connector/users.go @@ -31,10 +31,19 @@ type credentialUserBuilder struct { // has no ResourceDeleterV2: without the grant this connector cannot revoke // an org key, so it must not mint one either. offerOrgAPIKey bool + // offerServiceAccountAppKey follows the + // sync-service-account-application-keys grant, for the same reason: that + // grant is what registers the application-key syncer, and the syncer is + // what carries the deleter the SDK requires behind this descriptor. + offerServiceAccountAppKey bool } -func newCredentialUserBuilder(wrapper *client.DatadogClient, offerOrgAPIKey bool) *credentialUserBuilder { - return &credentialUserBuilder{userBuilder: newUserBuilder(wrapper), offerOrgAPIKey: offerOrgAPIKey} +func newCredentialUserBuilder(wrapper *client.DatadogClient, offerOrgAPIKey, offerServiceAccountAppKey bool) *credentialUserBuilder { + return &credentialUserBuilder{ + userBuilder: newUserBuilder(wrapper), + offerOrgAPIKey: offerOrgAPIKey, + offerServiceAccountAppKey: offerServiceAccountAppKey, + } } // IssueCapabilityDetails advertises the credential kinds this connector mints. @@ -52,14 +61,17 @@ func newCredentialUserBuilder(wrapper *client.DatadogClient, offerOrgAPIKey bool // issuance mapping with an honest owner, so it is the default when a caller // asks for the API_KEY shape without choosing a kind. func (u *credentialUserBuilder) IssueCapabilityDetails(context.Context) (*v2.CredentialDetailsCredentialIssue, annotations.Annotations, error) { - options := []*v2.CredentialIssueOptionDescriptor{ - v2.CredentialIssueOptionDescriptor_builder{ + options := []*v2.CredentialIssueOptionDescriptor{} + if u.offerServiceAccountAppKey { + options = append(options, v2.CredentialIssueOptionDescriptor_builder{ Option: v2.CapabilityDetailCredentialOption_CAPABILITY_DETAIL_CREDENTIAL_OPTION_API_KEY, ResourceMode: v2.CredentialResourceMode_CREDENTIAL_RESOURCE_MODE_DISCOVERABLE, SecretResourceTypeId: serviceAccountApplicationKeyResourceType.Id, CustomScopesAllowed: true, - Preferred: true, - }.Build(), + // Preferred only where it can be: exactly one descriptor per shape + // may set it, and it must be set whenever several share a shape. + Preferred: u.offerOrgAPIKey, + }.Build()) } if u.offerOrgAPIKey { options = append(options, v2.CredentialIssueOptionDescriptor_builder{ @@ -93,6 +105,10 @@ func (u *credentialUserBuilder) Issue(ctx context.Context, input *connectorbuild } switch secretResourceTypeID := input.CredentialOptions.GetSecretResourceTypeId(); secretResourceTypeID { case serviceAccountApplicationKeyResourceType.Id: + if !u.offerServiceAccountAppKey { + return nil, status.Error(codes.FailedPrecondition, + "baton-datadog: service account application key issuance requires sync-service-account-application-keys, which also provides the revoke path") + } return u.issueServiceAccountApplicationKey(ctx, input) case apiTokenResourceType.Id: if !u.offerOrgAPIKey { From 6dd74a35c55fe77d7eadc032fabd51c48608f3f5 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:44:16 +0000 Subject: [PATCH 47/49] fix(credentials): keep the owner lookup's failure code, and stop paging Two review findings on the owner-lookup fallback. Delete collapsed every non-NotFound failure from FindApplicationKeyOwner into InvalidArgument, discarding the classification wrapOfficialClientError exists to produce: a 500 is retryable, a 403 names a missing permission, and %v dropped the wrapped error with its rate-limit annotations. A transient blip therefore read as a terminal revoke failure and stranded the key -- the outcome the fallback was added to remove. Only the provider answering with no owner is unresolvable, and that case arrives as a plain error, so the code distinguishes them. The lookup also reaches an org-scoped endpoint the advertised permissions did not cover. Datadog governs GetApplicationKey under org_app_keys_read, not service_account_write, so a role holding exactly what this connector asked for would have 403'd on every revoke. Both permissions are now advertised and documented. Verified against the same OpenAPI spec the rest of these permissions came from, which also confirms the service-account endpoints need only service_account_write. Both name lookups now make one request instead of walking pages. The name searched for is always "c1-", so only a key whose own name contains that whole string can come back and a page of 100 cannot fill with them. Paging was answering a question the filter had already settled, and it was the source of the rate-limit concern raised in review. A full page now fails rather than reporting no match, because reporting no match would mint a duplicate of a key whose plaintext Datadog will not reissue. Co-authored-by: c1-squire-dev[bot] --- baton_capabilities.json | 6 ++ docs/connector.mdx | 2 +- pkg/client/client.go | 92 +++++++++++----------- pkg/client/client_test.go | 34 ++++---- pkg/connector/application_key.go | 13 ++- pkg/connector/credential_lifecycle_test.go | 35 ++++++++ pkg/connector/resource_types.go | 14 +++- 7 files changed, 126 insertions(+), 70 deletions(-) diff --git a/baton_capabilities.json b/baton_capabilities.json index 88b0f6e0..aa730e40 100644 --- a/baton_capabilities.json +++ b/baton_capabilities.json @@ -122,6 +122,9 @@ "permissions": [ { "permission": "service_account_write" + }, + { + "permission": "org_app_keys_read" } ] } @@ -136,6 +139,9 @@ "permissions": [ { "permission": "service_account_write" + }, + { + "permission": "org_app_keys_read" } ] } diff --git a/docs/connector.mdx b/docs/connector.mdx index fa32d2f6..e3f7ea46 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -54,7 +54,7 @@ Configuring the connector requires you to pass in credentials generated in Datad A user with the **Connector Administrator** or **Super Administrator** role in C1 and the **Datadog Admin** or **Datadog standard** role in Datadog must perform this task. -If your user has a custom Datadog role, make sure it includes **User App Keys**, **User Access Invite**, and **User Access Manage** to create, update, enable, and disable users from C1. If you enable **Sync secrets**, add **API Keys Read** to sync organization API keys. Add **Service Account Write**, which governs syncing, issuing, and revoking service account application keys, only if you also enable **Sync service account application keys**. Add **API Keys Write** and **API Keys Delete** only if you also enable **Allow organization API key deletion**; without that setting the connector never creates or deletes an organization API key, so those two permissions are not needed. **Service Account Write** is required, not optional, once that setting is on: a role that lacks it fails the sync rather than syncing an application-key inventory that is silently missing keys. That is why the setting is off by default — an existing install keeps syncing until the operator grants the permission. +If your user has a custom Datadog role, make sure it includes **User App Keys**, **User Access Invite**, and **User Access Manage** to create, update, enable, and disable users from C1. If you enable **Sync secrets**, add **API Keys Read** to sync organization API keys. Add **Service Account Write**, which governs syncing, issuing, and revoking service account application keys, only if you also enable **Sync service account application keys**; add **Org App Keys Read** alongside it, which the revoke path uses to find the service account that owns a key when the request does not name it. Add **API Keys Write** and **API Keys Delete** only if you also enable **Allow organization API key deletion**; without that setting the connector never creates or deletes an organization API key, so those two permissions are not needed. **Service Account Write** is required, not optional, once that setting is on: a role that lacks it fails the sync rather than syncing an application-key inventory that is silently missing keys. That is why the setting is off by default — an existing install keeps syncing until the operator grants the permission. ### Locate your Datadog site diff --git a/pkg/client/client.go b/pkg/client/client.go index bdb52b3a..652e0370 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -195,33 +195,41 @@ func (w *DatadogClient) CreateAPIKey(ctx context.Context, name string) (*IssuedA return &IssuedAPIKey{ID: *response.Data.Id, Secret: *response.Data.Attributes.Key}, nil } +// nameSearchPageSize is the page both name lookups ask for. It is a ceiling on +// how many keys a "c1-" filter may match, not a paging unit. +const nameSearchPageSize = int64(100) + // FindAPIKeyByName returns an exact name match, if one exists. Datadog's filter -// is a string search, so compare the returned name exactly before treating it as -// an existing issuance. +// is a substring search, so compare the returned name exactly before treating +// it as an existing issuance. +// +// One request, not a paged walk. The name searched for is always +// "c1-", so only a key whose own name contains that whole string +// can come back -- a page of 100 cannot fill with them. A full page therefore +// means the filter did not narrow the way this depends on, which is refused: +// reporting "no existing key" from a page that may not contain it would mint a +// duplicate, and Datadog cannot re-issue plaintext for the one that already +// exists. func (w *DatadogClient) FindAPIKeyByName(ctx context.Context, name string) (*datadogV2.PartialAPIKey, error) { ctx = w.withAuthContext(ctx) api := datadogV2.NewKeyManagementApi(w.officialClient) - const pageSize = int64(100) - const maxPages = int64(10000) - for page := int64(0); page < maxPages; page++ { - params := *datadogV2.NewListAPIKeysOptionalParameters().WithFilter(name).WithPageSize(pageSize).WithPageNumber(page) - response, httpRes, err := api.ListAPIKeys(ctx, params) - if httpRes != nil { - httpRes.Body.Close() - } - if err != nil { - return nil, wrapOfficialClientError("find API key by name", httpRes, err) - } - for _, key := range response.GetData() { - if key.Attributes != nil && key.Attributes.GetName() == name { - return &key, nil - } - } - if int64(len(response.GetData())) < pageSize { - return nil, nil + params := *datadogV2.NewListAPIKeysOptionalParameters().WithFilter(name).WithPageSize(nameSearchPageSize).WithPageNumber(0) + response, httpRes, err := api.ListAPIKeys(ctx, params) + if httpRes != nil { + defer httpRes.Body.Close() + } + if err != nil { + return nil, wrapOfficialClientError("find API key by name", httpRes, err) + } + for _, key := range response.GetData() { + if key.Attributes != nil && key.Attributes.GetName() == name { + return &key, nil } } - return nil, fmt.Errorf("find API key by name: exceeded %d pages", maxPages) + if int64(len(response.GetData())) >= nameSearchPageSize { + return nil, fmt.Errorf("find API key by name: filter %q returned a full page of %d keys with no exact match", name, nameSearchPageSize) + } + return nil, nil } func (w *DatadogClient) DeleteAPIKey(ctx context.Context, id string) error { @@ -307,36 +315,28 @@ func (w *DatadogClient) CreateServiceAccountApplicationKey(ctx context.Context, // FindServiceAccountApplicationKeyByName returns an exact name match among a // single service account's application keys, if one exists. Mirrors -// FindAPIKeyByName's exact-match-after-filter, paginated pattern, scoped to -// one service account instead of the whole org. +// FindAPIKeyByName, including its single-request shape and its refusal of a +// full page, scoped to one service account instead of the whole org. func (w *DatadogClient) FindServiceAccountApplicationKeyByName(ctx context.Context, serviceAccountID, name string) (*datadogV2.PartialApplicationKey, error) { ctx = w.withAuthContext(ctx) api := datadogV2.NewServiceAccountsApi(w.officialClient) - const pageSize = int64(100) - // maxPages bounds this loop so a provider that ignores page[number] and - // keeps returning full pages fails closed instead of spinning forever on - // the Issue hot path. 10_000 pages (1M keys) is far beyond any real - // service account's application-key count. - const maxPages = int64(10_000) - for page := int64(0); page < maxPages; page++ { - params := *datadogV2.NewListServiceAccountApplicationKeysOptionalParameters().WithFilter(name).WithPageSize(pageSize).WithPageNumber(page) - response, httpRes, err := api.ListServiceAccountApplicationKeys(ctx, serviceAccountID, params) - if httpRes != nil { - httpRes.Body.Close() - } - if err != nil { - return nil, wrapOfficialClientError("find service account application key by name", httpRes, err) - } - for _, key := range response.GetData() { - if key.Attributes != nil && key.Attributes.GetName() == name { - return &key, nil - } - } - if int64(len(response.GetData())) < pageSize { - return nil, nil + params := *datadogV2.NewListServiceAccountApplicationKeysOptionalParameters().WithFilter(name).WithPageSize(nameSearchPageSize).WithPageNumber(0) + response, httpRes, err := api.ListServiceAccountApplicationKeys(ctx, serviceAccountID, params) + if httpRes != nil { + defer httpRes.Body.Close() + } + if err != nil { + return nil, wrapOfficialClientError("find service account application key by name", httpRes, err) + } + for _, key := range response.GetData() { + if key.Attributes != nil && key.Attributes.GetName() == name { + return &key, nil } } - return nil, fmt.Errorf("find service account application key by name: exceeded %d pages without a short page", maxPages) + if int64(len(response.GetData())) >= nameSearchPageSize { + return nil, fmt.Errorf("find service account application key by name: filter %q returned a full page of %d keys with no exact match", name, nameSearchPageSize) + } + return nil, nil } // ListServiceAccountApplicationKeys lists every application key owned by the diff --git a/pkg/client/client_test.go b/pkg/client/client_test.go index 6ef20c79..8e38d7b1 100644 --- a/pkg/client/client_test.go +++ b/pkg/client/client_test.go @@ -92,31 +92,33 @@ func TestAPIKeyManagement(t *testing.T) { } }) - t.Run("find by name returns the exact match from a later page", func(t *testing.T) { + // A "c1-" filter cannot legitimately fill a page, so a full one + // means the filter did not narrow. Reporting "no existing key" from it + // would mint a duplicate of a key whose plaintext Datadog will not reissue, + // so it is an error rather than a not-found. + t.Run("find by name refuses a full page with no exact match", func(t *testing.T) { + requests := 0 server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + requests++ assertEqual(t, http.MethodGet, r.Method, "HTTP method should match") - q := r.URL.Query() - page := q.Get("page[number]") w.Header().Set("Content-Type", "application/json") - if page == "" || page == "0" { - // Page 0: 100 filler keys, none matching. - entries := make([]string, 100) - for i := range entries { - entries[i] = fmt.Sprintf(`{"id":"key-%d","type":"api_keys","attributes":{"name":"c1-other-%d"}}`, i, i) - } - _, _ = fmt.Fprintf(w, `{"data":[%s],"meta":{"page":{"total_filtered_count":101}}}`, strings.Join(entries, ",")) - return + entries := make([]string, 100) + for i := range entries { + entries[i] = fmt.Sprintf(`{"id":"key-%d","type":"api_keys","attributes":{"name":"c1-other-%d"}}`, i, i) } - // Page 1: one exact match. - _, _ = w.Write([]byte(`{"data":[{"id":"key-match","type":"api_keys","attributes":{"name":"c1-request"}}],"meta":{"page":{"total_filtered_count":101}}}`)) + _, _ = fmt.Fprintf(w, `{"data":[%s]}`, strings.Join(entries, ",")) })) defer server.Close() found, err := newOfficialTestClient(server.URL).FindAPIKeyByName(context.Background(), "c1-request") - assertNoError(t, err, "find API key by name should succeed") - assertNotNil(t, found, "expected an exact match across pages") if found != nil { - assertEqual(t, "key-match", found.GetId(), "should find the key on page 1, not page 0") + t.Fatal("a full page with no exact match is not a match") + } + if err == nil { + t.Fatal("expected a full page with no exact match to be refused") + } + if requests != 1 { + t.Fatalf("the lookup should make exactly one request, made %d", requests) } }) diff --git a/pkg/connector/application_key.go b/pkg/connector/application_key.go index 86daab4a..7ec4759f 100644 --- a/pkg/connector/application_key.go +++ b/pkg/connector/application_key.go @@ -87,11 +87,18 @@ func (o *applicationKeyBuilder) Delete(ctx context.Context, resourceID *v2.Resou if serviceAccountID == "" { owner, err := o.wrapper.FindApplicationKeyOwner(ctx, appKeyID) if err != nil { - if status.Code(err) == codes.NotFound { + switch status.Code(err) { + case codes.NotFound: return nil, nil + case codes.Unknown: + // The provider answered, and its answer names no owner. That + // is the only genuinely unresolvable case; every other code + // carries a retry or diagnosis signal worth preserving. + return nil, status.Errorf(codes.InvalidArgument, + "baton-datadog: the owning service account for application key %q could not be determined: %v", appKeyID, err) + default: + return nil, fmt.Errorf("baton-datadog: resolve owner for application key %q: %w", appKeyID, err) } - return nil, status.Errorf(codes.InvalidArgument, - "baton-datadog: the owning service account for application key %q could not be determined: %v", appKeyID, err) } serviceAccountID = owner } diff --git a/pkg/connector/credential_lifecycle_test.go b/pkg/connector/credential_lifecycle_test.go index 3c865391..c1db0e34 100644 --- a/pkg/connector/credential_lifecycle_test.go +++ b/pkg/connector/credential_lifecycle_test.go @@ -566,6 +566,41 @@ func TestApplicationKeyBuilderDeleteFailsWhenOwnerUnknown(t *testing.T) { require.Equal(t, codes.InvalidArgument, status.Code(err)) } +// TestApplicationKeyBuilderDeletePreservesOwnerLookupFailureCode: a transient +// failure of the owner lookup must not surface as a terminal InvalidArgument. +// wrapOfficialClientError classifies the provider response so a caller can +// retry or diagnose -- 500 is retryable, 403 names a missing permission. +// Collapsing either into InvalidArgument would strand the key being revoked. +func TestApplicationKeyBuilderDeletePreservesOwnerLookupFailureCode(t *testing.T) { + for _, tt := range []struct { + name string + status int + want codes.Code + }{ + {name: "provider 500", status: http.StatusInternalServerError, want: codes.Unavailable}, + {name: "provider 403", status: http.StatusForbidden, want: codes.PermissionDenied}, + } { + t.Run(tt.name, func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete { + t.Errorf("provider should not be asked to delete when the owner lookup failed") + } + w.WriteHeader(tt.status) + })) + defer server.Close() + + deleter := newApplicationKeyBuilder(newLifecycleTestWrapper(server.URL)) + _, err := deleter.Delete( + context.Background(), + &v2.ResourceId{ResourceType: serviceAccountApplicationKeyResourceType.Id, Resource: "appkey-transient-1"}, + nil, + ) + require.Error(t, err) + require.Equal(t, tt.want, status.Code(err)) + }) + } +} + // --- applicationKeyBuilder.List paging ------------------------------------ // newAppKeyListServer fakes the two endpoints applicationKeyBuilder.List diff --git a/pkg/connector/resource_types.go b/pkg/connector/resource_types.go index 8eaba058..3e32aaac 100644 --- a/pkg/connector/resource_types.go +++ b/pkg/connector/resource_types.go @@ -100,11 +100,17 @@ var ( // (organization API key) by resource type id, display name, and the // underlying SecretTrait's credential_detail (see application_key.go). // - // Every service-account application-key endpoint this connector calls - // requires exactly one Datadog RBAC permission, service_account_write: + // The service-account-scoped endpoints this connector calls require one + // Datadog RBAC permission, service_account_write: // ListServiceAccountApplicationKeys (sync), // CreateServiceAccountApplicationKey (issue) and - // DeleteServiceAccountApplicationKey (revoke). This is the "x-permission" + // DeleteServiceAccountApplicationKey (revoke). Revoke also reaches one + // org-scoped endpoint, GetApplicationKey, to recover the owning service + // account when the caller does not supply it; that one is governed by + // org_app_keys_read, so both are advertised. A role holding only + // service_account_write still syncs and issues -- it fails just the + // owner lookup, and reports PermissionDenied naming it. These are the + // "x-permission" // block Datadog publishes for each operation in its own OpenAPI spec // (docs.datadoghq.com/resources/json/full_spec_v2.json, the spec that // renders the public API reference); Datadog's role-permission page @@ -133,7 +139,7 @@ var ( Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_SECRET}, Annotations: annotations.New( &v2.SkipEntitlementsAndGrants{}, - capabilityPermissions("service_account_write"), + capabilityPermissions("service_account_write", "org_app_keys_read"), ), } scheduleResourceType = &v2.ResourceType{ From 2ff769bb799873c61e4fb509299c12b259d91aff Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:52:38 +0000 Subject: [PATCH 48/49] fix(lint): shorten the new flag's description, and align the docs to it revive rejected the description line at 213 characters. Shortened rather than wrapped, since the same string is the flag's help text and appears in README.md and config_schema.json. Two review points on the gate's documentation, both correct. The comment on userResourceType still said sync-secrets alone registers applicationKeyBuilder, which was the reasoning for scoping service_account_write to the application-key resource type -- that reasoning holds but now runs through two flags. And the capability table's application-key row carried no marker, so a reader scanning only the table would conclude those keys sync out of the box with Sync secrets. Co-authored-by: c1-squire-dev[bot] --- README.md | 2 +- config_schema.json | 2 +- docs/connector.mdx | 4 ++-- pkg/config/config.go | 2 +- pkg/connector/resource_types.go | 19 +++++++++++-------- 5 files changed, 16 insertions(+), 13 deletions(-) diff --git a/README.md b/README.md index 2289464b..ea4deb0c 100644 --- a/README.md +++ b/README.md @@ -96,7 +96,7 @@ Flags: --skip-full-sync This must be set to skip a full sync ($BATON_SKIP_FULL_SYNC) --sync-schedules Whether to sync on-call schedules or not ($BATON_SYNC_SCHEDULES) --sync-secrets Whether to sync secrets or not ($BATON_SYNC_SECRETS) - --sync-service-account-application-keys Sync, issue and revoke Datadog service account application keys. Off by default: requires the Datadog service_account_write permission, and a role without it fails the whole sync. ($BATON_SYNC_SERVICE_ACCOUNT_APPLICATION_KEYS) + --sync-service-account-application-keys Sync, issue and revoke Datadog service account application keys. Off by default: needs the service_account_write permission, without which the sync fails. ($BATON_SYNC_SERVICE_ACCOUNT_APPLICATION_KEYS) --ticketing This must be set to enable ticketing support ($BATON_TICKETING) -v, --version version for baton-datadog diff --git a/config_schema.json b/config_schema.json index 8970765c..ee7fc6ff 100644 --- a/config_schema.json +++ b/config_schema.json @@ -150,7 +150,7 @@ { "name": "sync-service-account-application-keys", "displayName": "Sync service account application keys", - "description": "Sync, issue and revoke Datadog service account application keys. Off by default: requires the Datadog service_account_write permission, and a role without it fails the whole sync.", + "description": "Sync, issue and revoke Datadog service account application keys. Off by default: needs the service_account_write permission, without which the sync fails.", "boolField": {} }, { diff --git a/docs/connector.mdx b/docs/connector.mdx index e3f7ea46..74176c2b 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -19,7 +19,7 @@ sidebarTitle: "Datadog" | Teams | | | | | | Schedules | * | | | | | Secrets - Organization API keys | | | ‡ | ‡ | -| Secrets - Service account application keys | | | | † | +| Secrets - Service account application keys | * | | * | *† | [This connector can sync secrets](/product/admin/inventory) and display them on the **Inventory** page. Organization API keys and service account application keys are synced, issued, and shown as distinct secret kinds: they are two different kinds of API key, not two spellings of one, and a request names which kind it wants. Service account application keys are the default kind. @@ -33,7 +33,7 @@ Revoking a service account application key needs the owning service account as w ‡Organization API key issuance and revocation both require **Allow organization API key deletion**, which is off by default and is separate from **Sync secrets**. An organization API key belongs to the whole organization rather than to the person it was issued to, and it cannot be scoped, so C1 will not mint one it has no permission to revoke. With the setting off, organization API keys still sync; they simply cannot be issued or deleted. -*Schedules and application keys are not enabled by default. Enable **Sync schedules** or **Sync secrets** plus **Sync service account application keys**, respectively, when configuring the connector. +*Schedules and service account application keys are not enabled by default. Enable **Sync schedules** for schedules; enable **Sync secrets** *and* **Sync service account application keys** for application keys — **Sync secrets** alone syncs organization API keys only. †Revoking a service account application key needs the owning service account as well as the key, because Datadog has no delete-by-key-id-alone form for these keys. When the request omits it, the connector looks the owner up from the key and proceeds; a key whose owner cannot be identified is refused rather than guessed at — see the note above. diff --git a/pkg/config/config.go b/pkg/config/config.go index d4a7ada3..0c541949 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -57,7 +57,7 @@ var ( // a credential absent from a completed sync reads as deleted. SyncServiceAccountApplicationKeys = field.BoolField( "sync-service-account-application-keys", - field.WithDescription("Sync, issue and revoke Datadog service account application keys. Off by default: requires the Datadog service_account_write permission, and a role without it fails the whole sync."), + field.WithDescription("Sync, issue and revoke Datadog service account application keys. Off by default: needs the service_account_write permission, without which the sync fails."), field.WithDefaultValue(false), field.WithDisplayName("Sync service account application keys"), ) diff --git a/pkg/connector/resource_types.go b/pkg/connector/resource_types.go index 3e32aaac..3734880d 100644 --- a/pkg/connector/resource_types.go +++ b/pkg/connector/resource_types.go @@ -21,21 +21,24 @@ var ( // user_access_manage for UpdateUser/DisableUser. // // service_account_write is deliberately NOT listed here even though - // userResourceType carries CAPABILITY_CREDENTIAL_ISSUE. Issuance only - // exists when sync-secrets is on -- that is the flag that swaps - // credentialUserBuilder in for userBuilder and registers - // applicationKeyBuilder at all -- so listing it here would tell every - // sync-secrets-off install to grant a Datadog Admin permission that no - // code path in that configuration can reach. + // userResourceType carries CAPABILITY_CREDENTIAL_ISSUE. Issuance of that + // kind exists only when sync-secrets and + // sync-service-account-application-keys are both on: the first is what + // swaps credentialUserBuilder in for userBuilder, the second is what + // registers applicationKeyBuilder and puts that kind in the descriptor + // list. Listing the permission here would tell every install without both + // to grant a Datadog Admin permission no code path in that configuration + // can reach. // - // baton_capabilities.json cannot express "only when sync-secrets is on": + // baton_capabilities.json cannot express "only under those flags": // it is one static document, generated by `./connector capabilities` from // a connector built with SyncSecrets and SyncSchedules forced true (see // cmd/baton-datadog/main.go), and CapabilityPermissions has no // conditional form. Scoping the permission to the resource type that only // exists under the flag is therefore how the conditionality is carried: // service_account_write lives on serviceAccountApplicationKeyResourceType, - // which is only registered when sync-secrets is on, and the user type's + // which is registered only when sync-secrets and + // sync-service-account-application-keys are both on, and the user type's // credential_issue block points at that type via secretResourceTypeId, so // the requirement is still discoverable from the metadata. userResourceType = &v2.ResourceType{ From e1fe609cdc575e9d1fe6dfd5e08f701a74ec1a49 Mon Sep 17 00:00:00 2001 From: Squire as Brandon High <759848+highb@users.noreply.github.com> Date: Fri, 28 Aug 2026 23:59:18 +0000 Subject: [PATCH 49/49] docs: add the application-key flag to the self-hosted example The self-hosted env-var block is the only place the connector's variable names are written out, and it named BATON_ALLOW_ORG_API_KEY_DELETION but not BATON_SYNC_SERVICE_ACCOUNT_APPLICATION_KEYS. An operator copying it got organization-key deletion and no application keys at all. Co-authored-by: c1-squire-dev[bot] --- docs/connector.mdx | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/connector.mdx b/docs/connector.mdx index 74176c2b..58aa6431 100644 --- a/docs/connector.mdx +++ b/docs/connector.mdx @@ -246,9 +246,15 @@ stringData: # Optional: include if you want to sync schedule data from Datadog BATON_SYNC_SCHEDULES: true - # Optional: include if you want to sync secrets (API keys) from Datadog + # Optional: include if you want to sync secrets (API keys) from Datadog. + # On its own this syncs organization API keys only. BATON_SYNC_SECRETS: true + # Optional: include alongside BATON_SYNC_SECRETS to sync, issue and revoke + # service account application keys. Needs the Datadog service_account_write + # and org_app_keys_read permissions; without them the sync fails. + BATON_SYNC_SERVICE_ACCOUNT_APPLICATION_KEYS: true + # Optional: include ONLY if you want C1 to issue and delete organization-wide # Datadog API keys. Syncing secrets does not grant this on its own. BATON_ALLOW_ORG_API_KEY_DELETION: true