From 3e91aad1c4d7e3f0a0549010a7cad23480d5fdc2 Mon Sep 17 00:00:00 2001 From: John Allers Date: Mon, 23 Mar 2026 12:44:05 -0400 Subject: [PATCH 1/3] [CXP-372] Add diagnostic logging for GitHub API errors When a sync fails with "Resource not accessible by integration", the existing logs don't identify which API call is failing. This adds structured warn logging and enriched error messages so the exact endpoint, HTTP status, and GitHub error details are captured. - Add gitHubErrorMessage() helper to extract method, URL, and sub-errors from *github.ErrorResponse - Enrich wrapGitHubError() auth/permission messages with GitHub error details - Add github_error field to graceful-skip warn logs in org, repository, and org_role syncers - Add warn logging to enterprise consumed licenses API error paths - Add warn logging to GraphQL SAML query error paths - Add warn logging to GitHub App installation token error paths --- pkg/connector/connector.go | 22 +++++++++++++++++++--- pkg/connector/enterprise_role.go | 9 +++++++++ pkg/connector/helpers.go | 30 ++++++++++++++++++++++++++++-- pkg/connector/org.go | 5 ++++- pkg/connector/org_role.go | 22 ++++++++++++++++++++-- pkg/connector/repository.go | 12 ++++++++++-- pkg/connector/user.go | 19 +++++++++++++++++-- 7 files changed, 107 insertions(+), 12 deletions(-) diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index da0d221f..747dc418 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -24,6 +24,7 @@ import ( "github.com/google/go-github/v69/github" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "github.com/shurcooL/githubv4" + "go.uber.org/zap" "golang.org/x/oauth2" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -445,20 +446,35 @@ func getJWTToken(appID string, privateKey string) (string, error) { func findInstallation(ctx context.Context, c *github.Client, orgName string) (*github.Installation, error) { installation, resp, err := c.Apps.FindOrganizationInstallation(ctx, orgName) if err != nil { - return nil, wrapGitHubError(err, resp, "github-connector: failed to find installation") + l := ctxzap.Extract(ctx) + l.Warn("failed to find GitHub App installation", + zap.String("org", orgName), + zap.String("github_error", gitHubErrorMessage(err)), + ) + return nil, wrapGitHubError(err, resp, fmt.Sprintf("github-connector: failed to find installation for org %s", orgName)) } return installation, nil } func getInstallationToken(ctx context.Context, c *github.Client, id int64) (*github.InstallationToken, error) { + l := ctxzap.Extract(ctx) token, resp, err := c.Apps.CreateInstallationToken(ctx, id, &github.InstallationTokenOptions{}) if err != nil { - return nil, err + l.Warn("failed to create GitHub App installation token", + zap.Int64("installation_id", id), + zap.String("github_error", gitHubErrorMessage(err)), + ) + return nil, fmt.Errorf("github-connector: failed to create installation token for installation %d: %w", id, err) } if resp.StatusCode != http.StatusCreated { body, _ := io.ReadAll(resp.Body) - return nil, fmt.Errorf("GitHub API error: %s", body) + l.Warn("unexpected status creating GitHub App installation token", + zap.Int64("installation_id", id), + zap.Int("http_status", resp.StatusCode), + zap.String("response_body", string(body)), + ) + return nil, fmt.Errorf("github-connector: unexpected status %d creating installation token for installation %d: %s", resp.StatusCode, id, body) } return token, nil diff --git a/pkg/connector/enterprise_role.go b/pkg/connector/enterprise_role.go index 172af02d..fab0c2c7 100644 --- a/pkg/connector/enterprise_role.go +++ b/pkg/connector/enterprise_role.go @@ -13,6 +13,8 @@ import ( resourceSdk "github.com/conductorone/baton-sdk/pkg/types/resource" "github.com/conductorone/baton-sdk/pkg/uhttp" "github.com/google/go-github/v69/github" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" "google.golang.org/grpc/codes" ) @@ -58,6 +60,13 @@ func (o *enterpriseRoleResourceType) fillCache(ctx context.Context) error { for continuePagination { consumedLicenses, _, err := o.customClient.ListEnterpriseConsumedLicenses(ctx, enterprise, page) if err != nil { + l := ctxzap.Extract(ctx) + l.Warn("failed to list enterprise consumed licenses", + zap.String("enterprise", enterprise), + zap.Int("page", page), + zap.String("endpoint", fmt.Sprintf("GET /enterprises/%s/consumed-licenses", enterprise)), + zap.Error(err), + ) return uhttp.WrapErrors(codes.PermissionDenied, fmt.Sprintf("baton-github: error listing enterprise consumed licenses for %s", enterprise), err) } diff --git a/pkg/connector/helpers.go b/pkg/connector/helpers.go index 76ce3ad0..7225cf86 100644 --- a/pkg/connector/helpers.go +++ b/pkg/connector/helpers.go @@ -303,6 +303,32 @@ func isTemporarilyUnavailable(resp *github.Response) bool { resp.StatusCode == http.StatusGatewayTimeout } +// gitHubErrorMessage extracts a human-readable error message from a GitHub API error. +// For *github.ErrorResponse, it includes the HTTP method, URL path, message, and sub-error details. +// For other error types, it returns err.Error(). +func gitHubErrorMessage(err error) string { + var ghErr *github.ErrorResponse + if errors.As(err, &ghErr) { + msg := ghErr.Message + if ghErr.Response != nil && ghErr.Response.Request != nil { + msg = fmt.Sprintf("%s %s: %s", ghErr.Response.Request.Method, ghErr.Response.Request.URL.Path, msg) + } + if len(ghErr.Errors) > 0 { + var errDetails []string + for _, e := range ghErr.Errors { + detail := e.Message + if detail == "" { + detail = fmt.Sprintf("resource=%s field=%s code=%s", e.Resource, e.Field, e.Code) + } + errDetails = append(errDetails, detail) + } + msg = fmt.Sprintf("%s [%s]", msg, strings.Join(errDetails, "; ")) + } + return msg + } + return err.Error() +} + // wrapGitHubError wraps GitHub API errors with appropriate gRPC status codes based on the HTTP response. // It handles rate limiting, authentication errors, permission errors, and generic errors. // The contextMsg parameter should describe the operation that failed (e.g., "failed to list teams"). @@ -337,10 +363,10 @@ func wrapGitHubError(err error, resp *github.Response, contextMsg string) error } if isAuthError(resp) { - return uhttp.WrapErrors(codes.Unauthenticated, contextMsg, err) + return uhttp.WrapErrors(codes.Unauthenticated, fmt.Sprintf("%s: %s", contextMsg, gitHubErrorMessage(err)), err) } if isPermissionError(resp) { - return uhttp.WrapErrors(codes.PermissionDenied, contextMsg, err) + return uhttp.WrapErrors(codes.PermissionDenied, fmt.Sprintf("%s: %s", contextMsg, gitHubErrorMessage(err)), err) } return fmt.Errorf("%s: %w", contextMsg, err) } diff --git a/pkg/connector/org.go b/pkg/connector/org.go index 59f1fb7b..34aad14a 100644 --- a/pkg/connector/org.go +++ b/pkg/connector/org.go @@ -126,7 +126,10 @@ func (o *orgResourceType) List( membership, resp, err := o.client.Organizations.GetOrgMembership(ctx, "", org.GetLogin()) if err != nil { if resp != nil && resp.StatusCode == http.StatusForbidden { - l.Warn("insufficient access to list org membership, skipping org", zap.String("org", org.GetLogin())) + l.Warn("insufficient access to list org membership, skipping org", + zap.String("org", org.GetLogin()), + zap.String("github_error", gitHubErrorMessage(err)), + ) continue } return nil, nil, wrapGitHubError(err, resp, "github-connector: failed to get org membership") diff --git a/pkg/connector/org_role.go b/pkg/connector/org_role.go index 32faf6c6..a71bee98 100644 --- a/pkg/connector/org_role.go +++ b/pkg/connector/org_role.go @@ -85,7 +85,12 @@ func (o *orgRoleResourceType) List( if err != nil { // Handle permission errors gracefully if resp != nil && (resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusNotFound) { - // Return empty list with no error to indicate we skipped this resource + l := ctxzap.Extract(ctx) + l.Warn("insufficient access to list organization roles, skipping", + zap.String("org", orgName), + zap.Int("http_status", resp.StatusCode), + zap.String("github_error", gitHubErrorMessage(err)), + ) return nil, &resourceSdk.SyncOpResults{}, nil } return nil, nil, wrapGitHubError(err, resp, "github-connector: failed to list organization roles") @@ -169,6 +174,13 @@ func (o *orgRoleResourceType) Grants( users, resp, err := o.client.Organizations.ListUsersAssignedToOrgRole(ctx, orgName, roleID, listOpts) if err != nil { if resp != nil && (resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusNotFound) { + l := ctxzap.Extract(ctx) + l.Warn("insufficient access to list users assigned to org role, skipping", + zap.String("org", orgName), + zap.Int64("role_id", roleID), + zap.Int("http_status", resp.StatusCode), + zap.String("github_error", gitHubErrorMessage(err)), + ) pageToken, err := bag.NextToken("") if err != nil { return nil, nil, err @@ -215,7 +227,13 @@ func (o *orgRoleResourceType) Grants( if err != nil { // Handle permission errors without erroring out. Some customers may not want to give us permissions to get org roles and members. if resp != nil && (resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusNotFound) { - // Return empty list with no error to indicate we skipped this resource + l := ctxzap.Extract(ctx) + l.Warn("insufficient access to list teams assigned to org role, skipping", + zap.String("org", orgName), + zap.Int64("role_id", roleID), + zap.Int("http_status", resp.StatusCode), + zap.String("github_error", gitHubErrorMessage(err)), + ) pageToken, err := bag.NextToken("") if err != nil { return nil, nil, err diff --git a/pkg/connector/repository.go b/pkg/connector/repository.go index 28546fe9..9aea826d 100644 --- a/pkg/connector/repository.go +++ b/pkg/connector/repository.go @@ -178,7 +178,11 @@ func (o *repositoryResourceType) Grants( users, resp, err := o.client.Repositories.ListCollaborators(ctx, orgName, resource.DisplayName, listOpts) if err != nil { if resp != nil && resp.StatusCode == http.StatusForbidden { - l.Warn("insufficient access to list collaborators", zap.String("repository", resource.DisplayName)) + l.Warn("insufficient access to list collaborators, skipping", + zap.String("org", orgName), + zap.String("repository", resource.DisplayName), + zap.String("github_error", gitHubErrorMessage(err)), + ) pageToken, err := skipGrantsForResourceType(bag) if err != nil { return nil, nil, err @@ -229,7 +233,11 @@ func (o *repositoryResourceType) Grants( teams, resp, err := o.client.Repositories.ListTeams(ctx, orgName, resource.DisplayName, listOpts) if err != nil { if resp != nil && resp.StatusCode == http.StatusForbidden { - l.Warn("insufficient access to list teams", zap.String("repository", resource.DisplayName)) + l.Warn("insufficient access to list teams for repository, skipping", + zap.String("org", orgName), + zap.String("repository", resource.DisplayName), + zap.String("github_error", gitHubErrorMessage(err)), + ) pageToken, err := skipGrantsForResourceType(bag) if err != nil { return nil, nil, err diff --git a/pkg/connector/user.go b/pkg/connector/user.go index 0043a158..e1107b30 100644 --- a/pkg/connector/user.go +++ b/pkg/connector/user.go @@ -198,7 +198,12 @@ func (o *userResourceType) List(ctx context.Context, parentID *v2.ResourceId, op l.Warn("failed to load enterprise email cache", zap.Error(loadErr)) } } else { - return nil, nil, err + l.Warn("GraphQL SAML identity query failed", + zap.String("org", orgName), + zap.String("user", u.GetLogin()), + zap.Error(err), + ) + return nil, nil, fmt.Errorf("baton-github: GraphQL SAML identity query failed for user %s in org %s: %w", u.GetLogin(), orgName, err) } } if err == nil && len(q.Organization.SamlIdentityProvider.ExternalIdentities.Edges) == 1 { @@ -362,6 +367,12 @@ func (o *userResourceType) loadEnterpriseEmailCache(ctx context.Context, ss sess for { consumedLicenses, _, err := o.customClient.ListEnterpriseConsumedLicenses(ctx, enterprise, page) if err != nil { + l.Warn("failed to fetch enterprise consumed licenses", + zap.String("enterprise", enterprise), + zap.Int("page", page), + zap.String("endpoint", fmt.Sprintf("GET /enterprises/%s/consumed-licenses", enterprise)), + zap.Error(err), + ) // Mark as loaded so we don't retry; partial data is still available. _ = session.SetJSON(ctx, ss, enterpriseEmailCacheLoadedKey, true) return fmt.Errorf("baton-github: failed to fetch enterprise consumed licenses for %s (page %d): %w", enterprise, page, err) @@ -440,7 +451,11 @@ func (o *userResourceType) hasSAML(ctx context.Context, orgName string, ss sessi } return false, nil } - return false, err + l.Warn("GraphQL SAML provider query failed", + zap.String("org", orgName), + zap.Error(err), + ) + return false, fmt.Errorf("baton-github: GraphQL SAML provider query failed for org %s: %w", orgName, err) } if q.Organization.SamlIdentityProvider.Id != "" { samlBool = true From 5d2e2e0434456f84d4b34f3f884685949f87184a Mon Sep 17 00:00:00 2001 From: John Allers Date: Mon, 23 Mar 2026 15:24:54 -0400 Subject: [PATCH 2/3] Downgrade high-volume skip-path logs from Warn to Debug Graceful-skip log paths fire per-resource when a GitHub App lacks permissions, producing excessive Warn volume in Datadog. These are expected behavior for limited-scope installations, not actionable warnings. Context remains available at LOG_LEVEL=debug. --- pkg/connector/org_role.go | 6 +++--- pkg/connector/repository.go | 4 ++-- pkg/connector/user.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pkg/connector/org_role.go b/pkg/connector/org_role.go index a71bee98..99fcf031 100644 --- a/pkg/connector/org_role.go +++ b/pkg/connector/org_role.go @@ -86,7 +86,7 @@ func (o *orgRoleResourceType) List( // Handle permission errors gracefully if resp != nil && (resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusNotFound) { l := ctxzap.Extract(ctx) - l.Warn("insufficient access to list organization roles, skipping", + l.Debug("insufficient access to list organization roles, skipping", zap.String("org", orgName), zap.Int("http_status", resp.StatusCode), zap.String("github_error", gitHubErrorMessage(err)), @@ -175,7 +175,7 @@ func (o *orgRoleResourceType) Grants( if err != nil { if resp != nil && (resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusNotFound) { l := ctxzap.Extract(ctx) - l.Warn("insufficient access to list users assigned to org role, skipping", + l.Debug("insufficient access to list users assigned to org role, skipping", zap.String("org", orgName), zap.Int64("role_id", roleID), zap.Int("http_status", resp.StatusCode), @@ -228,7 +228,7 @@ func (o *orgRoleResourceType) Grants( // Handle permission errors without erroring out. Some customers may not want to give us permissions to get org roles and members. if resp != nil && (resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusNotFound) { l := ctxzap.Extract(ctx) - l.Warn("insufficient access to list teams assigned to org role, skipping", + l.Debug("insufficient access to list teams assigned to org role, skipping", zap.String("org", orgName), zap.Int64("role_id", roleID), zap.Int("http_status", resp.StatusCode), diff --git a/pkg/connector/repository.go b/pkg/connector/repository.go index 9aea826d..571bb358 100644 --- a/pkg/connector/repository.go +++ b/pkg/connector/repository.go @@ -178,7 +178,7 @@ func (o *repositoryResourceType) Grants( users, resp, err := o.client.Repositories.ListCollaborators(ctx, orgName, resource.DisplayName, listOpts) if err != nil { if resp != nil && resp.StatusCode == http.StatusForbidden { - l.Warn("insufficient access to list collaborators, skipping", + l.Debug("insufficient access to list collaborators, skipping", zap.String("org", orgName), zap.String("repository", resource.DisplayName), zap.String("github_error", gitHubErrorMessage(err)), @@ -233,7 +233,7 @@ func (o *repositoryResourceType) Grants( teams, resp, err := o.client.Repositories.ListTeams(ctx, orgName, resource.DisplayName, listOpts) if err != nil { if resp != nil && resp.StatusCode == http.StatusForbidden { - l.Warn("insufficient access to list teams for repository, skipping", + l.Debug("insufficient access to list teams for repository, skipping", zap.String("org", orgName), zap.String("repository", resource.DisplayName), zap.String("github_error", gitHubErrorMessage(err)), diff --git a/pkg/connector/user.go b/pkg/connector/user.go index e1107b30..e90245d8 100644 --- a/pkg/connector/user.go +++ b/pkg/connector/user.go @@ -198,7 +198,7 @@ func (o *userResourceType) List(ctx context.Context, parentID *v2.ResourceId, op l.Warn("failed to load enterprise email cache", zap.Error(loadErr)) } } else { - l.Warn("GraphQL SAML identity query failed", + l.Debug("GraphQL SAML identity query failed", zap.String("org", orgName), zap.String("user", u.GetLogin()), zap.Error(err), From f7f2945a9de2d4bbec891386d299023527c21ff4 Mon Sep 17 00:00:00 2001 From: John Allers Date: Tue, 24 Mar 2026 06:51:27 -0400 Subject: [PATCH 3/3] Remove redundant warn logs; rely on enriched error messages wrapGitHubError now embeds gitHubErrorMessage detail, making pre-return Warn logs redundant. Also downgrades the org membership skip path from Warn to Debug for consistency with other graceful-skip paths. --- pkg/connector/connector.go | 5 ----- pkg/connector/enterprise_role.go | 9 --------- pkg/connector/org.go | 8 ++++---- pkg/connector/user.go | 5 ----- 4 files changed, 4 insertions(+), 23 deletions(-) diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index 747dc418..d035cedb 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -446,11 +446,6 @@ func getJWTToken(appID string, privateKey string) (string, error) { func findInstallation(ctx context.Context, c *github.Client, orgName string) (*github.Installation, error) { installation, resp, err := c.Apps.FindOrganizationInstallation(ctx, orgName) if err != nil { - l := ctxzap.Extract(ctx) - l.Warn("failed to find GitHub App installation", - zap.String("org", orgName), - zap.String("github_error", gitHubErrorMessage(err)), - ) return nil, wrapGitHubError(err, resp, fmt.Sprintf("github-connector: failed to find installation for org %s", orgName)) } return installation, nil diff --git a/pkg/connector/enterprise_role.go b/pkg/connector/enterprise_role.go index fab0c2c7..172af02d 100644 --- a/pkg/connector/enterprise_role.go +++ b/pkg/connector/enterprise_role.go @@ -13,8 +13,6 @@ import ( resourceSdk "github.com/conductorone/baton-sdk/pkg/types/resource" "github.com/conductorone/baton-sdk/pkg/uhttp" "github.com/google/go-github/v69/github" - "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" - "go.uber.org/zap" "google.golang.org/grpc/codes" ) @@ -60,13 +58,6 @@ func (o *enterpriseRoleResourceType) fillCache(ctx context.Context) error { for continuePagination { consumedLicenses, _, err := o.customClient.ListEnterpriseConsumedLicenses(ctx, enterprise, page) if err != nil { - l := ctxzap.Extract(ctx) - l.Warn("failed to list enterprise consumed licenses", - zap.String("enterprise", enterprise), - zap.Int("page", page), - zap.String("endpoint", fmt.Sprintf("GET /enterprises/%s/consumed-licenses", enterprise)), - zap.Error(err), - ) return uhttp.WrapErrors(codes.PermissionDenied, fmt.Sprintf("baton-github: error listing enterprise consumed licenses for %s", enterprise), err) } diff --git a/pkg/connector/org.go b/pkg/connector/org.go index 34aad14a..0f4c79bc 100644 --- a/pkg/connector/org.go +++ b/pkg/connector/org.go @@ -126,10 +126,10 @@ func (o *orgResourceType) List( membership, resp, err := o.client.Organizations.GetOrgMembership(ctx, "", org.GetLogin()) if err != nil { if resp != nil && resp.StatusCode == http.StatusForbidden { - l.Warn("insufficient access to list org membership, skipping org", - zap.String("org", org.GetLogin()), - zap.String("github_error", gitHubErrorMessage(err)), - ) + l.Debug("insufficient access to list org membership, skipping org", + zap.String("org", org.GetLogin()), + zap.String("github_error", gitHubErrorMessage(err)), + ) continue } return nil, nil, wrapGitHubError(err, resp, "github-connector: failed to get org membership") diff --git a/pkg/connector/user.go b/pkg/connector/user.go index e90245d8..83464133 100644 --- a/pkg/connector/user.go +++ b/pkg/connector/user.go @@ -198,11 +198,6 @@ func (o *userResourceType) List(ctx context.Context, parentID *v2.ResourceId, op l.Warn("failed to load enterprise email cache", zap.Error(loadErr)) } } else { - l.Debug("GraphQL SAML identity query failed", - zap.String("org", orgName), - zap.String("user", u.GetLogin()), - zap.Error(err), - ) return nil, nil, fmt.Errorf("baton-github: GraphQL SAML identity query failed for user %s in org %s: %w", u.GetLogin(), orgName, err) } }