From 241ff32bcb8ada6d5d2f8a64cda94f481066e21a Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Tue, 18 Aug 2026 16:42:12 -0300 Subject: [PATCH 01/13] fix: individual ctx for user-app lookup on usage --- pkg/connector/usage_event_feed.go | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/pkg/connector/usage_event_feed.go b/pkg/connector/usage_event_feed.go index 674c5a35..c29a54b8 100644 --- a/pkg/connector/usage_event_feed.go +++ b/pkg/connector/usage_event_feed.go @@ -13,6 +13,8 @@ import ( "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" "github.com/conductorone/baton-sdk/pkg/types/resource" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" reportsAdmin "google.golang.org/api/admin/reports/v1" "google.golang.org/api/googleapi" "google.golang.org/protobuf/types/known/timestamppb" @@ -27,7 +29,14 @@ var privateAppIDRegex = regexp.MustCompile("[0-9]{21}") // worry about — this only needs to cover the "does maxResults=1 return newest-first?" ordering // assumption from the acceptance criteria: with a >1 window, the true latest is picked // client-side regardless of Google's actual ordering. -const oauthAppLookupMaxResults = 5 +const oauthAppLookupMaxResults = 50 + +// oauthAppLookupLookback bounds the startTime of each per-(user, app) Reports API lookup. +const oauthAppLookupLookback = 180 * 24 * time.Hour + +// oauthAppLookupTimeout bounds how long a single (user, app) Reports API lookup — including its +// internal rate-limit wait and retries +const oauthAppLookupTimeout = 30 * time.Second type usageEventFeed struct { c *gwclient.GoogleWorkspaceClient @@ -96,9 +105,8 @@ func (f *usageEventFeed) lookupUser(ctx context.Context, client *gwclient.Google } return nil, fmt.Errorf("google-workspace: failed to list oauth tokens for %s: %w", user.Email, err) } - // Tokens.list can return multiple Token entries for the same client_id — e.g. a user - // granting a different scope set to the same app at different times. Dedupe here so each - // distinct app only ever triggers one Reports API lookup, never repeated ones. + // Tokens.list can return multiple Token entries for the same client_id. + // Dedupe here so each distinct app only ever triggers one Reports API lookup, never repeated ones. seenClientIDs := make(map[string]struct{}, len(tokenResp.Items)) events := make([]*v2.Event, 0, len(tokenResp.Items)) @@ -131,8 +139,21 @@ func (f *usageEventFeed) lookupUser(ctx context.Context, client *gwclient.Google // app (client_id), returning nil if there is no such activity within the lookup window. func (f *usageEventFeed) lookupAppLogin(ctx context.Context, client *gwclient.GoogleWorkspaceClient, user pendingUser, clientID, displayName string) (*v2.Event, error) { filters := "client_id==" + clientID - r, err := listActivitiesFilteredRateLimited(ctx, client, user.Email, "token", "authorize", "", "", filters, oauthAppLookupMaxResults) + startTime := time.Now().Add(-oauthAppLookupLookback).UTC().Format(time.RFC3339) + + lookupCtx, cancel := context.WithTimeout(ctx, oauthAppLookupTimeout) + defer cancel() + + r, err := listActivitiesFilteredRateLimited(lookupCtx, client, user.Email, "token", "authorize", startTime, "", filters, oauthAppLookupMaxResults) if err != nil { + if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { + // The per-lookup sub-deadline fired, not the caller's own context — this single + // (user, app) lookup was slow. Skip it rather than failing the whole batch so one + // slow call can't consume the entire sync's deadline. + ctxzap.Extract(ctx).Warn("google-workspace: timed out listing token activities, skipping", + zap.String("user", user.Email), zap.String("client_id", clientID)) + return nil, nil + } return nil, fmt.Errorf("google-workspace: failed to list token activities for %s app %s: %w", user.Email, clientID, err) } From 5feef6b16af97d3927ddf9a77b4717052fa16601 Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Tue, 18 Aug 2026 16:51:13 -0300 Subject: [PATCH 02/13] fix: bound event feed Reports API lookups to avoid DeadlineExceeded Each per-user (and per-app, for OAuth) Reports API activities.list call used no startTime and had no timeout of its own, so a single slow call could exceed the sync's overall deadline. Add a 180-day startTime bound (matches Google's Reports retention window and its documented guidance that narrower time ranges respond faster), a per-lookup sub-timeout that skips just that lookup instead of failing the whole batch, and widen maxResults to reduce pagination risk. Co-Authored-By: Claude Sonnet 5 --- pkg/connector/google_login_event_feed.go | 35 ++++++++++++++++++++---- pkg/connector/saml_event_feed.go | 26 +++++++++++++++++- pkg/connector/usage_event_feed.go | 19 ++++++------- 3 files changed, 62 insertions(+), 18 deletions(-) diff --git a/pkg/connector/google_login_event_feed.go b/pkg/connector/google_login_event_feed.go index d352abcc..7f67b699 100644 --- a/pkg/connector/google_login_event_feed.go +++ b/pkg/connector/google_login_event_feed.go @@ -2,13 +2,17 @@ package connector import ( "context" + "errors" "fmt" "strconv" + "time" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" "github.com/conductorone/baton-sdk/pkg/types/resource" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" reportsAdmin "google.golang.org/api/admin/reports/v1" "google.golang.org/protobuf/types/known/timestamppb" @@ -20,11 +24,18 @@ type bestActivity struct { occurredAt *timestamppb.Timestamp } -// googleLoginLookupMaxResults bounds the per-user Reports API lookup for Google Workspace -// sign-in events. There is exactly one target app (Google Workspace itself), so this only -// needs to cover the "does maxResults=1 return newest-first?" ordering uncertainty: fetch a -// small window and pick the maximum occurredAt client-side. -const googleLoginLookupMaxResults = 5 +// googleLoginLookupMaxResults bounds the per-user Reports API lookup; the true latest event is +// picked client-side, so this just needs to be large enough to avoid pagination. +const googleLoginLookupMaxResults = 50 + +// googleLoginLookupLookback bounds startTime so each lookup stays fast, per Google's guidance +// that narrower time ranges respond faster; 180 days matches Google's own Reports retention +// window. +const googleLoginLookupLookback = 180 * 24 * time.Hour + +// googleLoginLookupTimeout caps a single user's lookup so one slow call can't consume the +// whole sync's deadline. +const googleLoginLookupTimeout = 30 * time.Second // googleLoginEventFeed emits UsageEvents from Google Workspace sign-in activity. // Unlike SAML/OAuth feeds, the target resource is always Google Workspace itself. @@ -48,8 +59,20 @@ func (f *googleLoginEventFeed) EventFeedMetadata(_ context.Context) *v2.EventFee } func (f *googleLoginEventFeed) lookupUser(ctx context.Context, client *gwclient.GoogleWorkspaceClient, user pendingUser) ([]*v2.Event, error) { - r, err := listActivitiesRateLimited(ctx, client, user.Email, reportsAppLogin, "login_success", "", "", googleLoginLookupMaxResults) + startTime := time.Now().Add(-googleLoginLookupLookback).UTC().Format(time.RFC3339) + + lookupCtx, cancel := context.WithTimeout(ctx, googleLoginLookupTimeout) + defer cancel() + + r, err := listActivitiesRateLimited(lookupCtx, client, user.Email, reportsAppLogin, "login_success", startTime, "", googleLoginLookupMaxResults) if err != nil { + if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { + // Our sub-deadline fired, not the caller's context: skip this user instead of + // failing the whole batch. + ctxzap.Extract(ctx).Debug("google-workspace-connector: timed out listing google login activities, skipping", + zap.String("user", user.Email)) + return nil, nil + } return nil, fmt.Errorf("google-workspace-connector: failed to list google login activities for %s: %w", user.Email, err) } diff --git a/pkg/connector/saml_event_feed.go b/pkg/connector/saml_event_feed.go index ebae6761..fe7a41e1 100644 --- a/pkg/connector/saml_event_feed.go +++ b/pkg/connector/saml_event_feed.go @@ -2,13 +2,17 @@ package connector import ( "context" + "errors" "fmt" "strconv" + "time" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" "github.com/conductorone/baton-sdk/pkg/types/resource" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" reportsAdmin "google.golang.org/api/admin/reports/v1" "google.golang.org/protobuf/types/known/timestamppb" @@ -21,6 +25,14 @@ import ( // by resolved app ID, keeping only the newest event per app. const samlAppLookupMaxResults = 50 +// samlAppLookupLookback bounds startTime so each lookup stays fast, per Google's guidance that +// narrower time ranges respond faster; 180 days matches Google's own Reports retention window. +const samlAppLookupLookback = 180 * 24 * time.Hour + +// samlAppLookupTimeout caps a single user's lookup so one slow call can't consume the whole +// sync's deadline. +const samlAppLookupTimeout = 30 * time.Second + // samlEventFeed emits UsageEvents from Google Workspace SAML app login activity. type samlEventFeed struct { client *gwclient.GoogleWorkspaceClient @@ -54,8 +66,20 @@ type samlAppActivity struct { // authentication, so last login timestamps are accurate. SAML apps are identified by app name // (no numeric client_id). func (f *samlEventFeed) lookupUser(ctx context.Context, client *gwclient.GoogleWorkspaceClient, samlProfileMap map[string]string, user pendingUser) ([]*v2.Event, error) { - r, err := listActivitiesRateLimited(ctx, client, user.Email, reportsAppSAML, "login_success", "", "", samlAppLookupMaxResults) + startTime := time.Now().Add(-samlAppLookupLookback).UTC().Format(time.RFC3339) + + lookupCtx, cancel := context.WithTimeout(ctx, samlAppLookupTimeout) + defer cancel() + + r, err := listActivitiesRateLimited(lookupCtx, client, user.Email, reportsAppSAML, "login_success", startTime, "", samlAppLookupMaxResults) if err != nil { + if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { + // Our sub-deadline fired, not the caller's context: skip this user instead of + // failing the whole batch. + ctxzap.Extract(ctx).Debug("google-workspace-connector: timed out listing saml login activities, skipping", + zap.String("user", user.Email)) + return nil, nil + } return nil, fmt.Errorf("google-workspace-connector: failed to list saml login activities for %s: %w", user.Email, err) } diff --git a/pkg/connector/usage_event_feed.go b/pkg/connector/usage_event_feed.go index c29a54b8..0af6ec13 100644 --- a/pkg/connector/usage_event_feed.go +++ b/pkg/connector/usage_event_feed.go @@ -24,18 +24,16 @@ import ( var privateAppIDRegex = regexp.MustCompile("[0-9]{21}") -// oauthAppLookupMaxResults bounds each per-(user, app) Reports API lookup. Since the query is -// now scoped to one specific client_id via the `filters` param, there's no cross-app crowding to -// worry about — this only needs to cover the "does maxResults=1 return newest-first?" ordering -// assumption from the acceptance criteria: with a >1 window, the true latest is picked -// client-side regardless of Google's actual ordering. +// oauthAppLookupMaxResults bounds each per-(user, app) Reports API lookup; the true latest event +// is picked client-side, so this just needs to be large enough to avoid pagination. const oauthAppLookupMaxResults = 50 -// oauthAppLookupLookback bounds the startTime of each per-(user, app) Reports API lookup. +// oauthAppLookupLookback bounds startTime so each lookup stays fast, per Google's guidance that +// narrower time ranges respond faster; 180 days matches Google's own Reports retention window. const oauthAppLookupLookback = 180 * 24 * time.Hour -// oauthAppLookupTimeout bounds how long a single (user, app) Reports API lookup — including its -// internal rate-limit wait and retries +// oauthAppLookupTimeout caps a single (user, app) lookup so one slow call can't consume the +// whole sync's deadline. const oauthAppLookupTimeout = 30 * time.Second type usageEventFeed struct { @@ -147,9 +145,8 @@ func (f *usageEventFeed) lookupAppLogin(ctx context.Context, client *gwclient.Go r, err := listActivitiesFilteredRateLimited(lookupCtx, client, user.Email, "token", "authorize", startTime, "", filters, oauthAppLookupMaxResults) if err != nil { if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { - // The per-lookup sub-deadline fired, not the caller's own context — this single - // (user, app) lookup was slow. Skip it rather than failing the whole batch so one - // slow call can't consume the entire sync's deadline. + // Our sub-deadline fired, not the caller's context: skip this one app instead of + // failing the whole batch. ctxzap.Extract(ctx).Warn("google-workspace: timed out listing token activities, skipping", zap.String("user", user.Email), zap.String("client_id", clientID)) return nil, nil From 33291fc18ee731380aaa536e7350300913f9ba1c Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Tue, 18 Aug 2026 16:55:53 -0300 Subject: [PATCH 03/13] chore: promote new Debug logs to Warn, for visibility --- pkg/connector/google_login_event_feed.go | 2 +- pkg/connector/saml_event_feed.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/connector/google_login_event_feed.go b/pkg/connector/google_login_event_feed.go index 7f67b699..c8c14be9 100644 --- a/pkg/connector/google_login_event_feed.go +++ b/pkg/connector/google_login_event_feed.go @@ -69,7 +69,7 @@ func (f *googleLoginEventFeed) lookupUser(ctx context.Context, client *gwclient. if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { // Our sub-deadline fired, not the caller's context: skip this user instead of // failing the whole batch. - ctxzap.Extract(ctx).Debug("google-workspace-connector: timed out listing google login activities, skipping", + ctxzap.Extract(ctx).Warn("google-workspace-connector: timed out listing google login activities, skipping", zap.String("user", user.Email)) return nil, nil } diff --git a/pkg/connector/saml_event_feed.go b/pkg/connector/saml_event_feed.go index fe7a41e1..101c1dfa 100644 --- a/pkg/connector/saml_event_feed.go +++ b/pkg/connector/saml_event_feed.go @@ -76,7 +76,7 @@ func (f *samlEventFeed) lookupUser(ctx context.Context, client *gwclient.GoogleW if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { // Our sub-deadline fired, not the caller's context: skip this user instead of // failing the whole batch. - ctxzap.Extract(ctx).Debug("google-workspace-connector: timed out listing saml login activities, skipping", + ctxzap.Extract(ctx).Warn("google-workspace-connector: timed out listing saml login activities, skipping", zap.String("user", user.Email)) return nil, nil } From a39ea9b24fc14a14d4b78589b7747446fe05cf4c Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Wed, 19 Aug 2026 01:19:20 -0300 Subject: [PATCH 04/13] fix: scope per-attempt timeout to avoid swallowing retryable throttling The per-lookup sub-deadline previously wrapped the whole retry loop (quota wait + backoff + all attempts), so a persistently throttled 429/503 lookup could get cut off mid-backoff and be misclassified as "one slow call, skip silently" instead of surfacing as retryable throttling. Scope a 25s timeout to just the individual ListActivities call inside the retry loop, and raise each feed's per-lookup deadline to 45s so normal backoff (~31s worst case) has room to complete. Co-Authored-By: Claude Sonnet 5 --- pkg/connector/google_login_event_feed.go | 8 +++++--- pkg/connector/reports_rate_limiter.go | 16 ++++++++++++++-- pkg/connector/saml_event_feed.go | 8 +++++--- pkg/connector/usage_event_feed.go | 8 +++++--- 4 files changed, 29 insertions(+), 11 deletions(-) diff --git a/pkg/connector/google_login_event_feed.go b/pkg/connector/google_login_event_feed.go index c8c14be9..506f4482 100644 --- a/pkg/connector/google_login_event_feed.go +++ b/pkg/connector/google_login_event_feed.go @@ -33,9 +33,11 @@ const googleLoginLookupMaxResults = 50 // window. const googleLoginLookupLookback = 180 * 24 * time.Hour -// googleLoginLookupTimeout caps a single user's lookup so one slow call can't consume the -// whole sync's deadline. -const googleLoginLookupTimeout = 30 * time.Second +// googleLoginLookupTimeout caps a single user's lookup, including its retries, so one stuck +// lookup can't consume the whole sync's deadline. Kept above the retry loop's own worst-case +// backoff (~31s) so throttled 429/503s can complete their normal retries instead of being cut +// off mid-backoff. +const googleLoginLookupTimeout = 45 * time.Second // googleLoginEventFeed emits UsageEvents from Google Workspace sign-in activity. // Unlike SAML/OAuth feeds, the target resource is always Google Workspace itself. diff --git a/pkg/connector/reports_rate_limiter.go b/pkg/connector/reports_rate_limiter.go index 672fa134..e160438d 100644 --- a/pkg/connector/reports_rate_limiter.go +++ b/pkg/connector/reports_rate_limiter.go @@ -32,6 +32,11 @@ const ( reportsMaxRetries = 5 reportsInitialBackoff = 500 * time.Millisecond reportsMaxBackoff = 30 * time.Second + + // reportsPerAttemptTimeout bounds a single ListActivities call, not the retry loop as a + // whole, so a genuinely hung request is retried like any other transient error instead of + // being confused with the caller's overall lookup deadline expiring. + reportsPerAttemptTimeout = 25 * time.Second ) // reportsRateLimiter is a simple token-bucket limiter built on the standard library only @@ -119,11 +124,18 @@ func listActivitiesFilteredRateLimited( return nil, fmt.Errorf("google-workspace-connector: context cancelled waiting for reports api quota: %w", err) } - resp, err := client.ListActivities(ctx, userKey, applicationName, eventName, startTime, pageToken, filters, maxResults) + attemptCtx, cancel := context.WithTimeout(ctx, reportsPerAttemptTimeout) + resp, err := client.ListActivities(attemptCtx, userKey, applicationName, eventName, startTime, pageToken, filters, maxResults) + cancel() if err == nil { return resp, nil } - if attempt >= reportsMaxRetries || !isRetryableReportsError(err) { + + // ctx (not attemptCtx) still being live means it was attemptCtx's own shorter timeout + // that fired, not the caller's overall lookup deadline — treat that the same as a + // retryable 429/503 rather than as "out of time." + hungAttempt := errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil + if attempt >= reportsMaxRetries || !(isRetryableReportsError(err) || hungAttempt) { return nil, err } diff --git a/pkg/connector/saml_event_feed.go b/pkg/connector/saml_event_feed.go index 101c1dfa..431d57bc 100644 --- a/pkg/connector/saml_event_feed.go +++ b/pkg/connector/saml_event_feed.go @@ -29,9 +29,11 @@ const samlAppLookupMaxResults = 50 // narrower time ranges respond faster; 180 days matches Google's own Reports retention window. const samlAppLookupLookback = 180 * 24 * time.Hour -// samlAppLookupTimeout caps a single user's lookup so one slow call can't consume the whole -// sync's deadline. -const samlAppLookupTimeout = 30 * time.Second +// samlAppLookupTimeout caps a single user's lookup, including its retries, so one stuck lookup +// can't consume the whole sync's deadline. Kept above the retry loop's own worst-case backoff +// (~31s) so throttled 429/503s can complete their normal retries instead of being cut off +// mid-backoff. +const samlAppLookupTimeout = 45 * time.Second // samlEventFeed emits UsageEvents from Google Workspace SAML app login activity. type samlEventFeed struct { diff --git a/pkg/connector/usage_event_feed.go b/pkg/connector/usage_event_feed.go index 0af6ec13..1d46048d 100644 --- a/pkg/connector/usage_event_feed.go +++ b/pkg/connector/usage_event_feed.go @@ -32,9 +32,11 @@ const oauthAppLookupMaxResults = 50 // narrower time ranges respond faster; 180 days matches Google's own Reports retention window. const oauthAppLookupLookback = 180 * 24 * time.Hour -// oauthAppLookupTimeout caps a single (user, app) lookup so one slow call can't consume the -// whole sync's deadline. -const oauthAppLookupTimeout = 30 * time.Second +// oauthAppLookupTimeout caps a single (user, app) lookup, including its retries, so one stuck +// lookup can't consume the whole sync's deadline. Kept above the retry loop's own worst-case +// backoff (~31s) so throttled 429/503s can complete their normal retries instead of being cut +// off mid-backoff. +const oauthAppLookupTimeout = 45 * time.Second type usageEventFeed struct { c *gwclient.GoogleWorkspaceClient From 64858586f00418e9ec1be32bfc840f9b83914637 Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Wed, 19 Aug 2026 13:49:15 -0300 Subject: [PATCH 05/13] chore: fix lint error --- pkg/connector/reports_rate_limiter.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/connector/reports_rate_limiter.go b/pkg/connector/reports_rate_limiter.go index e160438d..8b103f62 100644 --- a/pkg/connector/reports_rate_limiter.go +++ b/pkg/connector/reports_rate_limiter.go @@ -135,7 +135,7 @@ func listActivitiesFilteredRateLimited( // that fired, not the caller's overall lookup deadline — treat that the same as a // retryable 429/503 rather than as "out of time." hungAttempt := errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil - if attempt >= reportsMaxRetries || !(isRetryableReportsError(err) || hungAttempt) { + if attempt >= reportsMaxRetries || (!isRetryableReportsError(err) && !hungAttempt) { return nil, err } From 408e9fcbfc5c71f5ec76ba95ad1f980deb7b46b7 Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Wed, 19 Aug 2026 14:35:43 -0300 Subject: [PATCH 06/13] fix: increase lookup timeout --- pkg/connector/google_login_event_feed.go | 9 +++++---- pkg/connector/saml_event_feed.go | 9 +++++---- pkg/connector/usage_event_feed.go | 9 +++++---- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/pkg/connector/google_login_event_feed.go b/pkg/connector/google_login_event_feed.go index 506f4482..bf2f6299 100644 --- a/pkg/connector/google_login_event_feed.go +++ b/pkg/connector/google_login_event_feed.go @@ -34,10 +34,11 @@ const googleLoginLookupMaxResults = 50 const googleLoginLookupLookback = 180 * 24 * time.Hour // googleLoginLookupTimeout caps a single user's lookup, including its retries, so one stuck -// lookup can't consume the whole sync's deadline. Kept above the retry loop's own worst-case -// backoff (~31s) so throttled 429/503s can complete their normal retries instead of being cut -// off mid-backoff. -const googleLoginLookupTimeout = 45 * time.Second +// lookup can't consume the whole sync's deadline. Kept above the worst case of a hung attempt +// (reportsPerAttemptTimeout) plus backoff plus a second full-length attempt (~51s), so the +// hung-attempt retry path in listActivitiesFilteredRateLimited has room to actually complete +// instead of being cut off by this deadline first. +const googleLoginLookupTimeout = 60 * time.Second // googleLoginEventFeed emits UsageEvents from Google Workspace sign-in activity. // Unlike SAML/OAuth feeds, the target resource is always Google Workspace itself. diff --git a/pkg/connector/saml_event_feed.go b/pkg/connector/saml_event_feed.go index 431d57bc..fe1cb688 100644 --- a/pkg/connector/saml_event_feed.go +++ b/pkg/connector/saml_event_feed.go @@ -30,10 +30,11 @@ const samlAppLookupMaxResults = 50 const samlAppLookupLookback = 180 * 24 * time.Hour // samlAppLookupTimeout caps a single user's lookup, including its retries, so one stuck lookup -// can't consume the whole sync's deadline. Kept above the retry loop's own worst-case backoff -// (~31s) so throttled 429/503s can complete their normal retries instead of being cut off -// mid-backoff. -const samlAppLookupTimeout = 45 * time.Second +// can't consume the whole sync's deadline. Kept above the worst case of a hung attempt +// (reportsPerAttemptTimeout) plus backoff plus a second full-length attempt (~51s), so the +// hung-attempt retry path in listActivitiesFilteredRateLimited has room to actually complete +// instead of being cut off by this deadline first. +const samlAppLookupTimeout = 60 * time.Second // samlEventFeed emits UsageEvents from Google Workspace SAML app login activity. type samlEventFeed struct { diff --git a/pkg/connector/usage_event_feed.go b/pkg/connector/usage_event_feed.go index 1d46048d..a5a7f02c 100644 --- a/pkg/connector/usage_event_feed.go +++ b/pkg/connector/usage_event_feed.go @@ -33,10 +33,11 @@ const oauthAppLookupMaxResults = 50 const oauthAppLookupLookback = 180 * 24 * time.Hour // oauthAppLookupTimeout caps a single (user, app) lookup, including its retries, so one stuck -// lookup can't consume the whole sync's deadline. Kept above the retry loop's own worst-case -// backoff (~31s) so throttled 429/503s can complete their normal retries instead of being cut -// off mid-backoff. -const oauthAppLookupTimeout = 45 * time.Second +// lookup can't consume the whole sync's deadline. Kept above the worst case of a hung attempt +// (reportsPerAttemptTimeout) plus backoff plus a second full-length attempt (~51s), so the +// hung-attempt retry path in listActivitiesFilteredRateLimited has room to actually complete +// instead of being cut off by this deadline first. +const oauthAppLookupTimeout = 60 * time.Second type usageEventFeed struct { c *gwclient.GoogleWorkspaceClient From c55843a2aa997159360fd163167f3a66424d200d Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Wed, 19 Aug 2026 15:15:43 -0300 Subject: [PATCH 07/13] fix: only impose per-attempt Reports timeout on callers with a deadline reportsPerAttemptTimeout previously applied to every caller of listActivitiesFilteredRateLimited, including app_login.go's unbounded, deadline-less lookups. There, every attempt timeout looked like a "hung attempt" and got retried, so a legitimately slow (but successful) call could burn all retries and fail outright. Extract the retry/backoff/timeout policy into retryListActivities, parameterized for testing, and only wrap attemptCtx with a sub-timeout when the caller's ctx already has a deadline. Add table-driven tests covering the hung-attempt-retry and caller-deadline-expired paths. Co-Authored-By: Claude Sonnet 5 --- pkg/connector/google_login_event_feed.go | 8 +- pkg/connector/reports_rate_limiter.go | 50 ++++-- pkg/connector/reports_rate_limiter_test.go | 197 +++++++++++++++++++++ pkg/connector/saml_event_feed.go | 8 +- pkg/connector/usage_event_feed.go | 8 +- 5 files changed, 245 insertions(+), 26 deletions(-) create mode 100644 pkg/connector/reports_rate_limiter_test.go diff --git a/pkg/connector/google_login_event_feed.go b/pkg/connector/google_login_event_feed.go index bf2f6299..ee337065 100644 --- a/pkg/connector/google_login_event_feed.go +++ b/pkg/connector/google_login_event_feed.go @@ -33,11 +33,9 @@ const googleLoginLookupMaxResults = 50 // window. const googleLoginLookupLookback = 180 * 24 * time.Hour -// googleLoginLookupTimeout caps a single user's lookup, including its retries, so one stuck -// lookup can't consume the whole sync's deadline. Kept above the worst case of a hung attempt -// (reportsPerAttemptTimeout) plus backoff plus a second full-length attempt (~51s), so the -// hung-attempt retry path in listActivitiesFilteredRateLimited has room to actually complete -// instead of being cut off by this deadline first. +// googleLoginLookupTimeout caps a single user's lookup, including retries. Kept above the worst +// case of a hung attempt plus backoff plus a full retry (~51s) so the hung-attempt retry in +// listActivitiesFilteredRateLimited has room to complete. const googleLoginLookupTimeout = 60 * time.Second // googleLoginEventFeed emits UsageEvents from Google Workspace sign-in activity. diff --git a/pkg/connector/reports_rate_limiter.go b/pkg/connector/reports_rate_limiter.go index 8b103f62..24b578ce 100644 --- a/pkg/connector/reports_rate_limiter.go +++ b/pkg/connector/reports_rate_limiter.go @@ -97,6 +97,10 @@ func (l *reportsRateLimiter) refillLocked() { l.lastRefill = l.now() } +// listActivitiesFunc mirrors GoogleWorkspaceClient.ListActivities so retryListActivities can be +// tested with a fake implementation. +type listActivitiesFunc func(ctx context.Context, userKey, applicationName, eventName, startTime, pageToken, filters string, maxResults int64) (*reportsAdmin.Activities, error) + // listActivitiesRateLimited waits for the shared filter-query budget, then calls // client.ListActivities, retrying with exponential backoff on 429/503 — both are transient, // SDK-retryable conditions, not connector bugs (see patterns-error-handling.md). @@ -118,24 +122,48 @@ func listActivitiesFilteredRateLimited( userKey, applicationName, eventName, startTime, pageToken, filters string, maxResults int64, ) (*reportsAdmin.Activities, error) { - backoff := reportsInitialBackoff + return retryListActivities( + ctx, sharedReportsRateLimiter, client.ListActivities, + reportsPerAttemptTimeout, reportsMaxRetries, reportsInitialBackoff, reportsMaxBackoff, + userKey, applicationName, eventName, startTime, pageToken, filters, maxResults, + ) +} + +// retryListActivities holds the retry/backoff/per-attempt-timeout policy, parameterized so tests +// can drive it with a fake call and short durations. The per-attempt timeout only applies when +// ctx already has a deadline, so deadline-less callers like app_login.go keep running unbounded. +func retryListActivities( + ctx context.Context, + limiter *reportsRateLimiter, + call listActivitiesFunc, + perAttemptTimeout time.Duration, + maxRetries int, + initialBackoff, maxBackoff time.Duration, + userKey, applicationName, eventName, startTime, pageToken, filters string, + maxResults int64, +) (*reportsAdmin.Activities, error) { + _, callerHasDeadline := ctx.Deadline() + backoff := initialBackoff for attempt := 0; ; attempt++ { - if err := sharedReportsRateLimiter.Wait(ctx); err != nil { + if err := limiter.Wait(ctx); err != nil { return nil, fmt.Errorf("google-workspace-connector: context cancelled waiting for reports api quota: %w", err) } - attemptCtx, cancel := context.WithTimeout(ctx, reportsPerAttemptTimeout) - resp, err := client.ListActivities(attemptCtx, userKey, applicationName, eventName, startTime, pageToken, filters, maxResults) + attemptCtx := ctx + cancel := func() {} + if callerHasDeadline { + attemptCtx, cancel = context.WithTimeout(ctx, perAttemptTimeout) + } + resp, err := call(attemptCtx, userKey, applicationName, eventName, startTime, pageToken, filters, maxResults) cancel() if err == nil { return resp, nil } - // ctx (not attemptCtx) still being live means it was attemptCtx's own shorter timeout - // that fired, not the caller's overall lookup deadline — treat that the same as a - // retryable 429/503 rather than as "out of time." - hungAttempt := errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil - if attempt >= reportsMaxRetries || (!isRetryableReportsError(err) && !hungAttempt) { + // ctx still being live means attemptCtx's own timeout fired, not the caller's deadline — + // treat that like a retryable 429/503 rather than "out of time." + hungAttempt := callerHasDeadline && errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil + if attempt >= maxRetries || (!isRetryableReportsError(err) && !hungAttempt) { return nil, err } @@ -146,8 +174,8 @@ func listActivitiesFilteredRateLimited( return nil, ctx.Err() } backoff *= 2 - if backoff > reportsMaxBackoff { - backoff = reportsMaxBackoff + if backoff > maxBackoff { + backoff = maxBackoff } } } diff --git a/pkg/connector/reports_rate_limiter_test.go b/pkg/connector/reports_rate_limiter_test.go new file mode 100644 index 00000000..fa824ba8 --- /dev/null +++ b/pkg/connector/reports_rate_limiter_test.go @@ -0,0 +1,197 @@ +package connector + +import ( + "context" + "errors" + "net/http" + "sync/atomic" + "testing" + "time" + + reportsAdmin "google.golang.org/api/admin/reports/v1" + "google.golang.org/api/googleapi" +) + +// unlimitedRateLimiter returns a limiter whose Wait() never blocks, so tests exercise only the +// retry/timeout policy in retryListActivities, not the token bucket. +func unlimitedRateLimiter() *reportsRateLimiter { + return newReportsRateLimiter(1_000_000) +} + +// blockingCall returns a listActivitiesFunc that, on each call, either sleeps for delay and +// returns results[callIndex] (nil error means success), or returns ctx.Err() if ctx is done +// first — mirroring how the real Reports API client responds to context cancellation. +func blockingCall(delay time.Duration, results ...error) (listActivitiesFunc, *int32) { + var calls int32 + fn := func(ctx context.Context, userKey, applicationName, eventName, startTime, pageToken, filters string, maxResults int64) (*reportsAdmin.Activities, error) { + idx := atomic.AddInt32(&calls, 1) - 1 + select { + case <-time.After(delay): + var err error + if int(idx) < len(results) { + err = results[idx] + } + if err != nil { + return nil, err + } + return &reportsAdmin.Activities{}, nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return fn, &calls +} + +// blockingCallWithDelays is blockingCall but each call gets its own delay from delays (the last +// entry repeats if there are more calls than delays), so a test can make the first attempt hang +// and later attempts return quickly. +func blockingCallWithDelays(delays []time.Duration, results ...error) (listActivitiesFunc, *int32) { + var calls int32 + fn := func(ctx context.Context, userKey, applicationName, eventName, startTime, pageToken, filters string, maxResults int64) (*reportsAdmin.Activities, error) { + idx := int(atomic.AddInt32(&calls, 1) - 1) + delay := delays[len(delays)-1] + if idx < len(delays) { + delay = delays[idx] + } + select { + case <-time.After(delay): + var err error + if idx < len(results) { + err = results[idx] + } + if err != nil { + return nil, err + } + return &reportsAdmin.Activities{}, nil + case <-ctx.Done(): + return nil, ctx.Err() + } + } + return fn, &calls +} + +func rateLimitedGoogleErr() error { + return &googleapi.Error{Code: http.StatusTooManyRequests} +} + +func TestRetryListActivities(t *testing.T) { + const ( + perAttemptTimeout = 20 * time.Millisecond + initialBackoff = 5 * time.Millisecond + maxBackoff = 20 * time.Millisecond + ) + + t.Run("succeeds on first attempt without retry", func(t *testing.T) { + call, calls := blockingCall(0) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + _, err := retryListActivities(ctx, unlimitedRateLimiter(), call, perAttemptTimeout, 5, initialBackoff, maxBackoff, "u", "app", "event", "", "", "", 10) + if err != nil { + t.Fatalf("expected success, got %v", err) + } + if got := atomic.LoadInt32(calls); got != 1 { + t.Fatalf("expected 1 call, got %d", got) + } + }) + + t.Run("retries a rate-limited error and then succeeds", func(t *testing.T) { + call, calls := blockingCall(0, rateLimitedGoogleErr()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + _, err := retryListActivities(ctx, unlimitedRateLimiter(), call, perAttemptTimeout, 5, initialBackoff, maxBackoff, "u", "app", "event", "", "", "", 10) + if err != nil { + t.Fatalf("expected eventual success, got %v", err) + } + if got := atomic.LoadInt32(calls); got != 2 { + t.Fatalf("expected 2 calls, got %d", got) + } + }) + + t.Run("fails immediately on a non-retryable error", func(t *testing.T) { + nonRetryable := &googleapi.Error{Code: http.StatusForbidden} + call, calls := blockingCall(0, nonRetryable, nil) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + _, err := retryListActivities(ctx, unlimitedRateLimiter(), call, perAttemptTimeout, 5, initialBackoff, maxBackoff, "u", "app", "event", "", "", "", 10) + if !errors.Is(err, nonRetryable) { + t.Fatalf("expected the non-retryable error back, got %v", err) + } + if got := atomic.LoadInt32(calls); got != 1 { + t.Fatalf("expected exactly 1 call (no retry), got %d", got) + } + }) + + t.Run("retries when only attemptCtx's own timeout fires (hung attempt)", func(t *testing.T) { + // First call sleeps past perAttemptTimeout so attemptCtx fires; the outer ctx has a much + // longer deadline and is still alive. Second call returns quickly and succeeds. + call, calls := blockingCallWithDelays([]time.Duration{perAttemptTimeout * 3, 0}) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + _, err := retryListActivities(ctx, unlimitedRateLimiter(), call, perAttemptTimeout, 5, initialBackoff, maxBackoff, "u", "app", "event", "", "", "", 10) + if err != nil { + t.Fatalf("expected the hung attempt to be retried and eventually succeed, got %v", err) + } + if got := atomic.LoadInt32(calls); got != 2 { + t.Fatalf("expected 2 calls (initial hung attempt + retry), got %d", got) + } + }) + + t.Run("returns immediately when the caller's own ctx is also done", func(t *testing.T) { + // Outer ctx deadline is shorter than perAttemptTimeout, so when attemptCtx fires it is + // because the caller's own deadline expired, not attemptCtx's independent timeout. + outerDeadline := perAttemptTimeout / 4 + call, calls := blockingCall(perAttemptTimeout * 3) + ctx, cancel := context.WithTimeout(context.Background(), outerDeadline) + defer cancel() + + _, err := retryListActivities(ctx, unlimitedRateLimiter(), call, perAttemptTimeout, 5, initialBackoff, maxBackoff, "u", "app", "event", "", "", "", 10) + if !errors.Is(err, context.DeadlineExceeded) { + t.Fatalf("expected context.DeadlineExceeded, got %v", err) + } + if got := atomic.LoadInt32(calls); got != 1 { + t.Fatalf("expected exactly 1 call (no retry once caller's ctx is done), got %d", got) + } + }) + + t.Run("imposes no per-attempt timeout when the caller has no deadline", func(t *testing.T) { + // A call that blocks well past perAttemptTimeout must still be allowed to complete when + // the caller (e.g. app_login.go) passed a ctx with no deadline of its own. + var sawDeadline bool + call := func(ctx context.Context, userKey, applicationName, eventName, startTime, pageToken, filters string, maxResults int64) (*reportsAdmin.Activities, error) { + _, sawDeadline = ctx.Deadline() + time.Sleep(perAttemptTimeout * 3) + return &reportsAdmin.Activities{}, nil + } + + start := time.Now() + _, err := retryListActivities(context.Background(), unlimitedRateLimiter(), call, perAttemptTimeout, 5, initialBackoff, maxBackoff, "u", "app", "event", "", "", "", 10) + if err != nil { + t.Fatalf("expected success, got %v", err) + } + if sawDeadline { + t.Fatalf("expected no deadline to be imposed on attemptCtx for a deadline-less caller") + } + if elapsed := time.Since(start); elapsed < perAttemptTimeout*3 { + t.Fatalf("expected the call to run past perAttemptTimeout uninterrupted, only took %v", elapsed) + } + }) + + t.Run("stops after maxRetries and returns the last error", func(t *testing.T) { + retryable := rateLimitedGoogleErr() + call, calls := blockingCall(0, retryable, retryable, retryable) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + _, err := retryListActivities(ctx, unlimitedRateLimiter(), call, perAttemptTimeout, 2, initialBackoff, maxBackoff, "u", "app", "event", "", "", "", 10) + if err == nil { + t.Fatalf("expected an error after exhausting retries") + } + if got := atomic.LoadInt32(calls); got != 3 { + t.Fatalf("expected 3 calls (1 initial + 2 retries), got %d", got) + } + }) +} diff --git a/pkg/connector/saml_event_feed.go b/pkg/connector/saml_event_feed.go index fe1cb688..96b9cdce 100644 --- a/pkg/connector/saml_event_feed.go +++ b/pkg/connector/saml_event_feed.go @@ -29,11 +29,9 @@ const samlAppLookupMaxResults = 50 // narrower time ranges respond faster; 180 days matches Google's own Reports retention window. const samlAppLookupLookback = 180 * 24 * time.Hour -// samlAppLookupTimeout caps a single user's lookup, including its retries, so one stuck lookup -// can't consume the whole sync's deadline. Kept above the worst case of a hung attempt -// (reportsPerAttemptTimeout) plus backoff plus a second full-length attempt (~51s), so the -// hung-attempt retry path in listActivitiesFilteredRateLimited has room to actually complete -// instead of being cut off by this deadline first. +// samlAppLookupTimeout caps a single user's lookup, including retries. Kept above the worst case +// of a hung attempt plus backoff plus a full retry (~51s) so the hung-attempt retry in +// listActivitiesFilteredRateLimited has room to complete. const samlAppLookupTimeout = 60 * time.Second // samlEventFeed emits UsageEvents from Google Workspace SAML app login activity. diff --git a/pkg/connector/usage_event_feed.go b/pkg/connector/usage_event_feed.go index a5a7f02c..b6fb791c 100644 --- a/pkg/connector/usage_event_feed.go +++ b/pkg/connector/usage_event_feed.go @@ -32,11 +32,9 @@ const oauthAppLookupMaxResults = 50 // narrower time ranges respond faster; 180 days matches Google's own Reports retention window. const oauthAppLookupLookback = 180 * 24 * time.Hour -// oauthAppLookupTimeout caps a single (user, app) lookup, including its retries, so one stuck -// lookup can't consume the whole sync's deadline. Kept above the worst case of a hung attempt -// (reportsPerAttemptTimeout) plus backoff plus a second full-length attempt (~51s), so the -// hung-attempt retry path in listActivitiesFilteredRateLimited has room to actually complete -// instead of being cut off by this deadline first. +// oauthAppLookupTimeout caps a single (user, app) lookup, including retries. Kept above the +// worst case of a hung attempt plus backoff plus a full retry (~51s) so the hung-attempt retry +// in listActivitiesFilteredRateLimited has room to complete. const oauthAppLookupTimeout = 60 * time.Second type usageEventFeed struct { From ac9596a11af72ebacf68c28b793508cc322ba35b Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Thu, 20 Aug 2026 13:09:05 -0300 Subject: [PATCH 08/13] fix: gate per-attempt Reports timeout on an explicit parameter, not ctx.Deadline() Inferring "does this caller want a per-attempt cap" from ctx.Deadline() was an implicit contract: it only exempted app_login.go's unbounded lookups because their ctx happens to have no deadline today. If the SDK ever attaches a deadline to the sync context, those callers would silently regain the 25s per-attempt cap and reintroduce the same DeadlineExceeded regression this fix removed. retryListActivities now takes perAttemptTimeout as an explicit parameter (0 = no cap) instead of inspecting ctx. Event feed callers opt in via new listActivitiesRateLimitedBounded / listActivitiesFilteredRateLimitedBounded wrappers that pass reportsPerAttemptTimeout explicitly; app_login.go's existing listActivitiesRateLimited / listActivitiesFilteredRateLimited calls now explicitly pass 0, so they stay unbounded regardless of what ctx carries. Co-Authored-By: Claude Sonnet 5 --- pkg/connector/google_login_event_feed.go | 4 +- pkg/connector/reports_rate_limiter.go | 43 +++++++++-- pkg/connector/reports_rate_limiter_test.go | 90 ++++++++++++++++++++-- pkg/connector/saml_event_feed.go | 4 +- pkg/connector/usage_event_feed.go | 4 +- 5 files changed, 125 insertions(+), 20 deletions(-) diff --git a/pkg/connector/google_login_event_feed.go b/pkg/connector/google_login_event_feed.go index ee337065..31a3689e 100644 --- a/pkg/connector/google_login_event_feed.go +++ b/pkg/connector/google_login_event_feed.go @@ -35,7 +35,7 @@ const googleLoginLookupLookback = 180 * 24 * time.Hour // googleLoginLookupTimeout caps a single user's lookup, including retries. Kept above the worst // case of a hung attempt plus backoff plus a full retry (~51s) so the hung-attempt retry in -// listActivitiesFilteredRateLimited has room to complete. +// listActivitiesRateLimitedBounded has room to complete. const googleLoginLookupTimeout = 60 * time.Second // googleLoginEventFeed emits UsageEvents from Google Workspace sign-in activity. @@ -65,7 +65,7 @@ func (f *googleLoginEventFeed) lookupUser(ctx context.Context, client *gwclient. lookupCtx, cancel := context.WithTimeout(ctx, googleLoginLookupTimeout) defer cancel() - r, err := listActivitiesRateLimited(lookupCtx, client, user.Email, reportsAppLogin, "login_success", startTime, "", googleLoginLookupMaxResults) + r, err := listActivitiesRateLimitedBounded(lookupCtx, client, user.Email, reportsAppLogin, "login_success", startTime, "", googleLoginLookupMaxResults) if err != nil { if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { // Our sub-deadline fired, not the caller's context: skip this user instead of diff --git a/pkg/connector/reports_rate_limiter.go b/pkg/connector/reports_rate_limiter.go index 24b578ce..ef937294 100644 --- a/pkg/connector/reports_rate_limiter.go +++ b/pkg/connector/reports_rate_limiter.go @@ -103,7 +103,9 @@ type listActivitiesFunc func(ctx context.Context, userKey, applicationName, even // listActivitiesRateLimited waits for the shared filter-query budget, then calls // client.ListActivities, retrying with exponential backoff on 429/503 — both are transient, -// SDK-retryable conditions, not connector bugs (see patterns-error-handling.md). +// SDK-retryable conditions, not connector bugs (see patterns-error-handling.md). It imposes no +// per-attempt timeout: use listActivitiesRateLimitedBounded for callers that set their own +// lookupCtx sub-deadline and want a hung attempt retried instead of just waited out. func listActivitiesRateLimited( ctx context.Context, client *gwclient.GoogleWorkspaceClient, @@ -121,6 +123,34 @@ func listActivitiesFilteredRateLimited( client *gwclient.GoogleWorkspaceClient, userKey, applicationName, eventName, startTime, pageToken, filters string, maxResults int64, +) (*reportsAdmin.Activities, error) { + return retryListActivities( + ctx, sharedReportsRateLimiter, client.ListActivities, + 0, reportsMaxRetries, reportsInitialBackoff, reportsMaxBackoff, + userKey, applicationName, eventName, startTime, pageToken, filters, maxResults, + ) +} + +// listActivitiesRateLimitedBounded is listActivitiesRateLimited plus reportsPerAttemptTimeout +// applied to every attempt. Reserved for the event feed callers (usage/google-login/saml), which +// always wrap the call in their own bounded lookupCtx, so a hung attempt can be told apart from +// the caller's own deadline expiring and retried instead of failing the lookup outright. +func listActivitiesRateLimitedBounded( + ctx context.Context, + client *gwclient.GoogleWorkspaceClient, + userKey, applicationName, eventName, startTime, pageToken string, + maxResults int64, +) (*reportsAdmin.Activities, error) { + return listActivitiesFilteredRateLimitedBounded(ctx, client, userKey, applicationName, eventName, startTime, pageToken, "", maxResults) +} + +// listActivitiesFilteredRateLimitedBounded is listActivitiesRateLimitedBounded plus an optional +// Reports API `filters` expression; see listActivitiesFilteredRateLimited. +func listActivitiesFilteredRateLimitedBounded( + ctx context.Context, + client *gwclient.GoogleWorkspaceClient, + userKey, applicationName, eventName, startTime, pageToken, filters string, + maxResults int64, ) (*reportsAdmin.Activities, error) { return retryListActivities( ctx, sharedReportsRateLimiter, client.ListActivities, @@ -130,8 +160,9 @@ func listActivitiesFilteredRateLimited( } // retryListActivities holds the retry/backoff/per-attempt-timeout policy, parameterized so tests -// can drive it with a fake call and short durations. The per-attempt timeout only applies when -// ctx already has a deadline, so deadline-less callers like app_login.go keep running unbounded. +// can drive it with a fake call and short durations. perAttemptTimeout == 0 means "no per-attempt +// cap" — the caller's own ctx is used as-is and a DeadlineExceeded from it is never retried as +// a hung attempt. func retryListActivities( ctx context.Context, limiter *reportsRateLimiter, @@ -142,7 +173,7 @@ func retryListActivities( userKey, applicationName, eventName, startTime, pageToken, filters string, maxResults int64, ) (*reportsAdmin.Activities, error) { - _, callerHasDeadline := ctx.Deadline() + applyPerAttemptTimeout := perAttemptTimeout > 0 backoff := initialBackoff for attempt := 0; ; attempt++ { if err := limiter.Wait(ctx); err != nil { @@ -151,7 +182,7 @@ func retryListActivities( attemptCtx := ctx cancel := func() {} - if callerHasDeadline { + if applyPerAttemptTimeout { attemptCtx, cancel = context.WithTimeout(ctx, perAttemptTimeout) } resp, err := call(attemptCtx, userKey, applicationName, eventName, startTime, pageToken, filters, maxResults) @@ -162,7 +193,7 @@ func retryListActivities( // ctx still being live means attemptCtx's own timeout fired, not the caller's deadline — // treat that like a retryable 429/503 rather than "out of time." - hungAttempt := callerHasDeadline && errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil + hungAttempt := applyPerAttemptTimeout && errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil if attempt >= maxRetries || (!isRetryableReportsError(err) && !hungAttempt) { return nil, err } diff --git a/pkg/connector/reports_rate_limiter_test.go b/pkg/connector/reports_rate_limiter_test.go index fa824ba8..253dd7cd 100644 --- a/pkg/connector/reports_rate_limiter_test.go +++ b/pkg/connector/reports_rate_limiter_test.go @@ -4,12 +4,15 @@ import ( "context" "errors" "net/http" + "net/url" "sync/atomic" "testing" "time" reportsAdmin "google.golang.org/api/admin/reports/v1" "google.golang.org/api/googleapi" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // unlimitedRateLimiter returns a limiter whose Wait() never blocks, so tests exercise only the @@ -74,6 +77,34 @@ func rateLimitedGoogleErr() error { return &googleapi.Error{Code: http.StatusTooManyRequests} } +// productionWrappedRateLimitErr mirrors what client.ListActivities actually returns for a 429: +// wrapGoogleApiErrorWithContext joins a gRPC status with the original *googleapi.Error, rather +// than returning the bare *googleapi.Error the other tests use. +func productionWrappedRateLimitErr() error { + return errors.Join(status.Error(codes.Unavailable, "rate limited"), &googleapi.Error{Code: http.StatusTooManyRequests}) +} + +// blockingCallWrappedTimeout is blockingCallWithDelays, but on a ctx timeout it returns the +// error shape production code actually produces (a *url.Error wrapping the context error, as +// net/http's transport does) instead of a bare context error. +func blockingCallWrappedTimeout(delays []time.Duration) (listActivitiesFunc, *int32) { + var calls int32 + fn := func(ctx context.Context, userKey, applicationName, eventName, startTime, pageToken, filters string, maxResults int64) (*reportsAdmin.Activities, error) { + idx := int(atomic.AddInt32(&calls, 1) - 1) + delay := delays[len(delays)-1] + if idx < len(delays) { + delay = delays[idx] + } + select { + case <-time.After(delay): + return &reportsAdmin.Activities{}, nil + case <-ctx.Done(): + return nil, &url.Error{Op: "Get", URL: "https://example.com", Err: ctx.Err()} + } + } + return fn, &calls +} + func TestRetryListActivities(t *testing.T) { const ( perAttemptTimeout = 20 * time.Millisecond @@ -157,29 +188,72 @@ func TestRetryListActivities(t *testing.T) { } }) - t.Run("imposes no per-attempt timeout when the caller has no deadline", func(t *testing.T) { - // A call that blocks well past perAttemptTimeout must still be allowed to complete when - // the caller (e.g. app_login.go) passed a ctx with no deadline of its own. - var sawDeadline bool + t.Run("imposes no per-attempt timeout when perAttemptTimeout is 0, even with a deadline-bearing ctx", func(t *testing.T) { + // perAttemptTimeout == 0 must disable the sub-timeout on its own terms, regardless of + // whether ctx happens to carry a deadline (e.g. a future SDK change attaches one to the + // sync context) — this is what listActivitiesFilteredRateLimited (used by app_login.go) + // relies on to stay unbounded. + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + wantDeadline, _ := ctx.Deadline() + + var gotDeadline time.Time + var gotHasDeadline bool call := func(ctx context.Context, userKey, applicationName, eventName, startTime, pageToken, filters string, maxResults int64) (*reportsAdmin.Activities, error) { - _, sawDeadline = ctx.Deadline() + gotDeadline, gotHasDeadline = ctx.Deadline() time.Sleep(perAttemptTimeout * 3) return &reportsAdmin.Activities{}, nil } start := time.Now() - _, err := retryListActivities(context.Background(), unlimitedRateLimiter(), call, perAttemptTimeout, 5, initialBackoff, maxBackoff, "u", "app", "event", "", "", "", 10) + _, err := retryListActivities(ctx, unlimitedRateLimiter(), call, 0, 5, initialBackoff, maxBackoff, "u", "app", "event", "", "", "", 10) if err != nil { t.Fatalf("expected success, got %v", err) } - if sawDeadline { - t.Fatalf("expected no deadline to be imposed on attemptCtx for a deadline-less caller") + // attemptCtx must be ctx itself (same deadline as the caller's own, not a new shorter + // one), proving perAttemptTimeout==0 disabled the sub-timeout rather than the caller + // simply having no deadline to begin with. + if !gotHasDeadline || !gotDeadline.Equal(wantDeadline) { + t.Fatalf("expected attemptCtx to carry the caller's own deadline unchanged, got hasDeadline=%v deadline=%v want=%v", gotHasDeadline, gotDeadline, wantDeadline) } if elapsed := time.Since(start); elapsed < perAttemptTimeout*3 { t.Fatalf("expected the call to run past perAttemptTimeout uninterrupted, only took %v", elapsed) } }) + t.Run("retries a rate-limited error wrapped the way production code wraps it", func(t *testing.T) { + // Pins the contract that isRetryableReportsError's errors.As still finds the + // *googleapi.Error inside wrapGoogleApiErrorWithContext's errors.Join, not just a bare + // *googleapi.Error. + call, calls := blockingCall(0, productionWrappedRateLimitErr()) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + _, err := retryListActivities(ctx, unlimitedRateLimiter(), call, perAttemptTimeout, 5, initialBackoff, maxBackoff, "u", "app", "event", "", "", "", 10) + if err != nil { + t.Fatalf("expected eventual success, got %v", err) + } + if got := atomic.LoadInt32(calls); got != 2 { + t.Fatalf("expected 2 calls, got %d", got) + } + }) + + t.Run("retries a hung attempt whose timeout arrives wrapped like production's transport error", func(t *testing.T) { + // Pins the contract that hungAttempt's errors.Is still finds context.DeadlineExceeded + // inside a *url.Error, not just a bare context error. + call, calls := blockingCallWrappedTimeout([]time.Duration{perAttemptTimeout * 3, 0}) + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + _, err := retryListActivities(ctx, unlimitedRateLimiter(), call, perAttemptTimeout, 5, initialBackoff, maxBackoff, "u", "app", "event", "", "", "", 10) + if err != nil { + t.Fatalf("expected the wrapped hung attempt to be retried and eventually succeed, got %v", err) + } + if got := atomic.LoadInt32(calls); got != 2 { + t.Fatalf("expected 2 calls (initial hung attempt + retry), got %d", got) + } + }) + t.Run("stops after maxRetries and returns the last error", func(t *testing.T) { retryable := rateLimitedGoogleErr() call, calls := blockingCall(0, retryable, retryable, retryable) diff --git a/pkg/connector/saml_event_feed.go b/pkg/connector/saml_event_feed.go index 96b9cdce..4cd6a83a 100644 --- a/pkg/connector/saml_event_feed.go +++ b/pkg/connector/saml_event_feed.go @@ -31,7 +31,7 @@ const samlAppLookupLookback = 180 * 24 * time.Hour // samlAppLookupTimeout caps a single user's lookup, including retries. Kept above the worst case // of a hung attempt plus backoff plus a full retry (~51s) so the hung-attempt retry in -// listActivitiesFilteredRateLimited has room to complete. +// listActivitiesRateLimitedBounded has room to complete. const samlAppLookupTimeout = 60 * time.Second // samlEventFeed emits UsageEvents from Google Workspace SAML app login activity. @@ -72,7 +72,7 @@ func (f *samlEventFeed) lookupUser(ctx context.Context, client *gwclient.GoogleW lookupCtx, cancel := context.WithTimeout(ctx, samlAppLookupTimeout) defer cancel() - r, err := listActivitiesRateLimited(lookupCtx, client, user.Email, reportsAppSAML, "login_success", startTime, "", samlAppLookupMaxResults) + r, err := listActivitiesRateLimitedBounded(lookupCtx, client, user.Email, reportsAppSAML, "login_success", startTime, "", samlAppLookupMaxResults) if err != nil { if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { // Our sub-deadline fired, not the caller's context: skip this user instead of diff --git a/pkg/connector/usage_event_feed.go b/pkg/connector/usage_event_feed.go index b6fb791c..6324b4ab 100644 --- a/pkg/connector/usage_event_feed.go +++ b/pkg/connector/usage_event_feed.go @@ -34,7 +34,7 @@ const oauthAppLookupLookback = 180 * 24 * time.Hour // oauthAppLookupTimeout caps a single (user, app) lookup, including retries. Kept above the // worst case of a hung attempt plus backoff plus a full retry (~51s) so the hung-attempt retry -// in listActivitiesFilteredRateLimited has room to complete. +// in listActivitiesFilteredRateLimitedBounded has room to complete. const oauthAppLookupTimeout = 60 * time.Second type usageEventFeed struct { @@ -143,7 +143,7 @@ func (f *usageEventFeed) lookupAppLogin(ctx context.Context, client *gwclient.Go lookupCtx, cancel := context.WithTimeout(ctx, oauthAppLookupTimeout) defer cancel() - r, err := listActivitiesFilteredRateLimited(lookupCtx, client, user.Email, "token", "authorize", startTime, "", filters, oauthAppLookupMaxResults) + r, err := listActivitiesFilteredRateLimitedBounded(lookupCtx, client, user.Email, "token", "authorize", startTime, "", filters, oauthAppLookupMaxResults) if err != nil { if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { // Our sub-deadline fired, not the caller's context: skip this one app instead of From 9bbf031c8dd28243d8180909274bb97e1062b968 Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Mon, 24 Aug 2026 13:06:02 -0300 Subject: [PATCH 09/13] chore: downgrade Warn logs to Debug logs for skipped user-app activity --- pkg/connector/google_login_event_feed.go | 3 ++- pkg/connector/saml_event_feed.go | 3 ++- pkg/connector/usage_event_feed.go | 3 ++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/connector/google_login_event_feed.go b/pkg/connector/google_login_event_feed.go index 31a3689e..371427d5 100644 --- a/pkg/connector/google_login_event_feed.go +++ b/pkg/connector/google_login_event_feed.go @@ -60,6 +60,7 @@ func (f *googleLoginEventFeed) EventFeedMetadata(_ context.Context) *v2.EventFee } func (f *googleLoginEventFeed) lookupUser(ctx context.Context, client *gwclient.GoogleWorkspaceClient, user pendingUser) ([]*v2.Event, error) { + l := ctxzap.Extract(ctx) startTime := time.Now().Add(-googleLoginLookupLookback).UTC().Format(time.RFC3339) lookupCtx, cancel := context.WithTimeout(ctx, googleLoginLookupTimeout) @@ -70,7 +71,7 @@ func (f *googleLoginEventFeed) lookupUser(ctx context.Context, client *gwclient. if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { // Our sub-deadline fired, not the caller's context: skip this user instead of // failing the whole batch. - ctxzap.Extract(ctx).Warn("google-workspace-connector: timed out listing google login activities, skipping", + l.Debug("google-workspace-connector: timed out listing google login activities, skipping", zap.String("user", user.Email)) return nil, nil } diff --git a/pkg/connector/saml_event_feed.go b/pkg/connector/saml_event_feed.go index 4cd6a83a..17ecba87 100644 --- a/pkg/connector/saml_event_feed.go +++ b/pkg/connector/saml_event_feed.go @@ -67,6 +67,7 @@ type samlAppActivity struct { // authentication, so last login timestamps are accurate. SAML apps are identified by app name // (no numeric client_id). func (f *samlEventFeed) lookupUser(ctx context.Context, client *gwclient.GoogleWorkspaceClient, samlProfileMap map[string]string, user pendingUser) ([]*v2.Event, error) { + l := ctxzap.Extract(ctx) startTime := time.Now().Add(-samlAppLookupLookback).UTC().Format(time.RFC3339) lookupCtx, cancel := context.WithTimeout(ctx, samlAppLookupTimeout) @@ -77,7 +78,7 @@ func (f *samlEventFeed) lookupUser(ctx context.Context, client *gwclient.GoogleW if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { // Our sub-deadline fired, not the caller's context: skip this user instead of // failing the whole batch. - ctxzap.Extract(ctx).Warn("google-workspace-connector: timed out listing saml login activities, skipping", + l.Debug("google-workspace-connector: timed out listing saml login activities, skipping", zap.String("user", user.Email)) return nil, nil } diff --git a/pkg/connector/usage_event_feed.go b/pkg/connector/usage_event_feed.go index 6324b4ab..6f736163 100644 --- a/pkg/connector/usage_event_feed.go +++ b/pkg/connector/usage_event_feed.go @@ -137,6 +137,7 @@ func (f *usageEventFeed) lookupUser(ctx context.Context, client *gwclient.Google // lookupAppLogin fetches this user's most recent "authorize" activity for one specific OAuth // app (client_id), returning nil if there is no such activity within the lookup window. func (f *usageEventFeed) lookupAppLogin(ctx context.Context, client *gwclient.GoogleWorkspaceClient, user pendingUser, clientID, displayName string) (*v2.Event, error) { + l := ctxzap.Extract(ctx) filters := "client_id==" + clientID startTime := time.Now().Add(-oauthAppLookupLookback).UTC().Format(time.RFC3339) @@ -148,7 +149,7 @@ func (f *usageEventFeed) lookupAppLogin(ctx context.Context, client *gwclient.Go if errors.Is(err, context.DeadlineExceeded) && ctx.Err() == nil { // Our sub-deadline fired, not the caller's context: skip this one app instead of // failing the whole batch. - ctxzap.Extract(ctx).Warn("google-workspace: timed out listing token activities, skipping", + l.Debug("google-workspace: timed out listing token activities, skipping", zap.String("user", user.Email), zap.String("client_id", clientID)) return nil, nil } From 278535e88d740cf4eeea86720c7402612f80918b Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Mon, 24 Aug 2026 13:13:53 -0300 Subject: [PATCH 10/13] chore: duplicated consts merged into one --- pkg/connector/google_login_event_feed.go | 7 +------ pkg/connector/reports_rate_limiter.go | 3 +++ pkg/connector/saml_event_feed.go | 6 +----- pkg/connector/usage_event_feed.go | 6 +----- 4 files changed, 6 insertions(+), 16 deletions(-) diff --git a/pkg/connector/google_login_event_feed.go b/pkg/connector/google_login_event_feed.go index 371427d5..9cfb916f 100644 --- a/pkg/connector/google_login_event_feed.go +++ b/pkg/connector/google_login_event_feed.go @@ -28,11 +28,6 @@ type bestActivity struct { // picked client-side, so this just needs to be large enough to avoid pagination. const googleLoginLookupMaxResults = 50 -// googleLoginLookupLookback bounds startTime so each lookup stays fast, per Google's guidance -// that narrower time ranges respond faster; 180 days matches Google's own Reports retention -// window. -const googleLoginLookupLookback = 180 * 24 * time.Hour - // googleLoginLookupTimeout caps a single user's lookup, including retries. Kept above the worst // case of a hung attempt plus backoff plus a full retry (~51s) so the hung-attempt retry in // listActivitiesRateLimitedBounded has room to complete. @@ -61,7 +56,7 @@ func (f *googleLoginEventFeed) EventFeedMetadata(_ context.Context) *v2.EventFee func (f *googleLoginEventFeed) lookupUser(ctx context.Context, client *gwclient.GoogleWorkspaceClient, user pendingUser) ([]*v2.Event, error) { l := ctxzap.Extract(ctx) - startTime := time.Now().Add(-googleLoginLookupLookback).UTC().Format(time.RFC3339) + startTime := time.Now().Add(-reportsLookback).UTC().Format(time.RFC3339) lookupCtx, cancel := context.WithTimeout(ctx, googleLoginLookupTimeout) defer cancel() diff --git a/pkg/connector/reports_rate_limiter.go b/pkg/connector/reports_rate_limiter.go index ef937294..cd5c695c 100644 --- a/pkg/connector/reports_rate_limiter.go +++ b/pkg/connector/reports_rate_limiter.go @@ -37,6 +37,9 @@ const ( // whole, so a genuinely hung request is retried like any other transient error instead of // being confused with the caller's overall lookup deadline expiring. reportsPerAttemptTimeout = 25 * time.Second + + // reportsLookback matches Google's Reports retention window; shared by the event feeds. + reportsLookback = 180 * 24 * time.Hour ) // reportsRateLimiter is a simple token-bucket limiter built on the standard library only diff --git a/pkg/connector/saml_event_feed.go b/pkg/connector/saml_event_feed.go index 17ecba87..85553de3 100644 --- a/pkg/connector/saml_event_feed.go +++ b/pkg/connector/saml_event_feed.go @@ -25,10 +25,6 @@ import ( // by resolved app ID, keeping only the newest event per app. const samlAppLookupMaxResults = 50 -// samlAppLookupLookback bounds startTime so each lookup stays fast, per Google's guidance that -// narrower time ranges respond faster; 180 days matches Google's own Reports retention window. -const samlAppLookupLookback = 180 * 24 * time.Hour - // samlAppLookupTimeout caps a single user's lookup, including retries. Kept above the worst case // of a hung attempt plus backoff plus a full retry (~51s) so the hung-attempt retry in // listActivitiesRateLimitedBounded has room to complete. @@ -68,7 +64,7 @@ type samlAppActivity struct { // (no numeric client_id). func (f *samlEventFeed) lookupUser(ctx context.Context, client *gwclient.GoogleWorkspaceClient, samlProfileMap map[string]string, user pendingUser) ([]*v2.Event, error) { l := ctxzap.Extract(ctx) - startTime := time.Now().Add(-samlAppLookupLookback).UTC().Format(time.RFC3339) + startTime := time.Now().Add(-reportsLookback).UTC().Format(time.RFC3339) lookupCtx, cancel := context.WithTimeout(ctx, samlAppLookupTimeout) defer cancel() diff --git a/pkg/connector/usage_event_feed.go b/pkg/connector/usage_event_feed.go index 6f736163..1c1ecd4e 100644 --- a/pkg/connector/usage_event_feed.go +++ b/pkg/connector/usage_event_feed.go @@ -28,10 +28,6 @@ var privateAppIDRegex = regexp.MustCompile("[0-9]{21}") // is picked client-side, so this just needs to be large enough to avoid pagination. const oauthAppLookupMaxResults = 50 -// oauthAppLookupLookback bounds startTime so each lookup stays fast, per Google's guidance that -// narrower time ranges respond faster; 180 days matches Google's own Reports retention window. -const oauthAppLookupLookback = 180 * 24 * time.Hour - // oauthAppLookupTimeout caps a single (user, app) lookup, including retries. Kept above the // worst case of a hung attempt plus backoff plus a full retry (~51s) so the hung-attempt retry // in listActivitiesFilteredRateLimitedBounded has room to complete. @@ -139,7 +135,7 @@ func (f *usageEventFeed) lookupUser(ctx context.Context, client *gwclient.Google func (f *usageEventFeed) lookupAppLogin(ctx context.Context, client *gwclient.GoogleWorkspaceClient, user pendingUser, clientID, displayName string) (*v2.Event, error) { l := ctxzap.Extract(ctx) filters := "client_id==" + clientID - startTime := time.Now().Add(-oauthAppLookupLookback).UTC().Format(time.RFC3339) + startTime := time.Now().Add(-reportsLookback).UTC().Format(time.RFC3339) lookupCtx, cancel := context.WithTimeout(ctx, oauthAppLookupTimeout) defer cancel() From d7370078f828576525d5b2b35ad75b8ce18cf072 Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Mon, 24 Aug 2026 13:18:30 -0300 Subject: [PATCH 11/13] refactor: collapse listActivitiesFilteredRateLimited into listActivitiesRateLimited It had shrunk to a single caller (its own unfiltered sibling), so inline the retryListActivities call directly and drop the extra layer. Co-Authored-By: Claude Sonnet 5 --- pkg/connector/reports_rate_limiter.go | 17 +++-------------- pkg/connector/reports_rate_limiter_test.go | 4 ++-- 2 files changed, 5 insertions(+), 16 deletions(-) diff --git a/pkg/connector/reports_rate_limiter.go b/pkg/connector/reports_rate_limiter.go index cd5c695c..aa158921 100644 --- a/pkg/connector/reports_rate_limiter.go +++ b/pkg/connector/reports_rate_limiter.go @@ -114,23 +114,11 @@ func listActivitiesRateLimited( client *gwclient.GoogleWorkspaceClient, userKey, applicationName, eventName, startTime, pageToken string, maxResults int64, -) (*reportsAdmin.Activities, error) { - return listActivitiesFilteredRateLimited(ctx, client, userKey, applicationName, eventName, startTime, pageToken, "", maxResults) -} - -// listActivitiesFilteredRateLimited is listActivitiesRateLimited plus an optional Reports API -// `filters` expression (e.g. "client_id=="), for callers that need to scope a lookup to one -// specific app rather than an entire app-type. -func listActivitiesFilteredRateLimited( - ctx context.Context, - client *gwclient.GoogleWorkspaceClient, - userKey, applicationName, eventName, startTime, pageToken, filters string, - maxResults int64, ) (*reportsAdmin.Activities, error) { return retryListActivities( ctx, sharedReportsRateLimiter, client.ListActivities, 0, reportsMaxRetries, reportsInitialBackoff, reportsMaxBackoff, - userKey, applicationName, eventName, startTime, pageToken, filters, maxResults, + userKey, applicationName, eventName, startTime, pageToken, "", maxResults, ) } @@ -148,7 +136,8 @@ func listActivitiesRateLimitedBounded( } // listActivitiesFilteredRateLimitedBounded is listActivitiesRateLimitedBounded plus an optional -// Reports API `filters` expression; see listActivitiesFilteredRateLimited. +// Reports API `filters` expression (e.g. "client_id=="), for callers that need to scope a +// lookup to one specific app rather than an entire app-type. func listActivitiesFilteredRateLimitedBounded( ctx context.Context, client *gwclient.GoogleWorkspaceClient, diff --git a/pkg/connector/reports_rate_limiter_test.go b/pkg/connector/reports_rate_limiter_test.go index 253dd7cd..aa24f1e0 100644 --- a/pkg/connector/reports_rate_limiter_test.go +++ b/pkg/connector/reports_rate_limiter_test.go @@ -191,8 +191,8 @@ func TestRetryListActivities(t *testing.T) { t.Run("imposes no per-attempt timeout when perAttemptTimeout is 0, even with a deadline-bearing ctx", func(t *testing.T) { // perAttemptTimeout == 0 must disable the sub-timeout on its own terms, regardless of // whether ctx happens to carry a deadline (e.g. a future SDK change attaches one to the - // sync context) — this is what listActivitiesFilteredRateLimited (used by app_login.go) - // relies on to stay unbounded. + // sync context) — this is what listActivitiesRateLimited (used by app_login.go) relies + // on to stay unbounded. ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() wantDeadline, _ := ctx.Deadline() From 314bb9718b95a7ff4c9c241d2b7f0acf656a9d8a Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Wed, 26 Aug 2026 00:41:48 -0300 Subject: [PATCH 12/13] chore: improve docs --- pkg/connector/event_feed_common.go | 5 ++--- pkg/connector/usage_event_feed.go | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pkg/connector/event_feed_common.go b/pkg/connector/event_feed_common.go index f16dc7fd..70740d4a 100644 --- a/pkg/connector/event_feed_common.go +++ b/pkg/connector/event_feed_common.go @@ -9,9 +9,8 @@ // pagination.StreamToken, and never loops internally across directory pages (see // ref-antipatterns.md, "Client-Side Pagination Loop"). // -// Ordering note: it is not documented whether activities.list returns newest-first when no -// startTime/orderBy is given, so each lookup fetches a small bounded window (not maxResults=1) -// and picks the maximum occurredAt client-side, per the plan's acceptance-criteria fallback. +// Ordering note: activities.list ordering is undocumented, so with startTime=180 days back and +// maxResults=50, each lookup still picks the maximum occurredAt client-side rather than trusting result order. package connector import ( diff --git a/pkg/connector/usage_event_feed.go b/pkg/connector/usage_event_feed.go index 1c1ecd4e..178275ba 100644 --- a/pkg/connector/usage_event_feed.go +++ b/pkg/connector/usage_event_feed.go @@ -131,7 +131,7 @@ func (f *usageEventFeed) lookupUser(ctx context.Context, client *gwclient.Google } // lookupAppLogin fetches this user's most recent "authorize" activity for one specific OAuth -// app (client_id), returning nil if there is no such activity within the lookup window. +// app (client_id), returning nil if there is no such activity, or if the lookup timed out. func (f *usageEventFeed) lookupAppLogin(ctx context.Context, client *gwclient.GoogleWorkspaceClient, user pendingUser, clientID, displayName string) (*v2.Event, error) { l := ctxzap.Extract(ctx) filters := "client_id==" + clientID From 66b1ff21354e39be21a0831fb7e98ee2146e1d19 Mon Sep 17 00:00:00 2001 From: Javier David Carnelli Date: Wed, 26 Aug 2026 00:48:11 -0300 Subject: [PATCH 13/13] test: give blockingCallWithDelays an error-wrapping hook, drop duplicate helper blockingCallWrappedTimeout was a near-copy of blockingCallWithDelays that only differed in how it wrapped a ctx-done error, and it dropped the per-call results parameter. Add blockingCallWithDelaysWrapped(wrapCtxErr) so both timeout-wrapping shapes and per-call results are expressible from one helper, and delete the duplicate. Co-Authored-By: Claude Sonnet 5 --- pkg/connector/reports_rate_limiter_test.go | 37 ++++++++-------------- 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/pkg/connector/reports_rate_limiter_test.go b/pkg/connector/reports_rate_limiter_test.go index aa24f1e0..9185093d 100644 --- a/pkg/connector/reports_rate_limiter_test.go +++ b/pkg/connector/reports_rate_limiter_test.go @@ -47,8 +47,16 @@ func blockingCall(delay time.Duration, results ...error) (listActivitiesFunc, *i // blockingCallWithDelays is blockingCall but each call gets its own delay from delays (the last // entry repeats if there are more calls than delays), so a test can make the first attempt hang -// and later attempts return quickly. +// and later attempts return quickly. On a ctx timeout, the returned error is ctx.Err() as-is; use +// blockingCallWithDelaysWrapped to pin a production-shaped wrapping of that error instead. func blockingCallWithDelays(delays []time.Duration, results ...error) (listActivitiesFunc, *int32) { + return blockingCallWithDelaysWrapped(delays, func(err error) error { return err }, results...) +} + +// blockingCallWithDelaysWrapped is blockingCallWithDelays, but passes a ctx timeout's error +// through wrapCtxErr before returning it — e.g. to mirror how net/http's transport wraps a +// context error in a *url.Error, so tests can pin that errors.Is still unwraps it correctly. +func blockingCallWithDelaysWrapped(delays []time.Duration, wrapCtxErr func(error) error, results ...error) (listActivitiesFunc, *int32) { var calls int32 fn := func(ctx context.Context, userKey, applicationName, eventName, startTime, pageToken, filters string, maxResults int64) (*reportsAdmin.Activities, error) { idx := int(atomic.AddInt32(&calls, 1) - 1) @@ -67,7 +75,7 @@ func blockingCallWithDelays(delays []time.Duration, results ...error) (listActiv } return &reportsAdmin.Activities{}, nil case <-ctx.Done(): - return nil, ctx.Err() + return nil, wrapCtxErr(ctx.Err()) } } return fn, &calls @@ -84,27 +92,6 @@ func productionWrappedRateLimitErr() error { return errors.Join(status.Error(codes.Unavailable, "rate limited"), &googleapi.Error{Code: http.StatusTooManyRequests}) } -// blockingCallWrappedTimeout is blockingCallWithDelays, but on a ctx timeout it returns the -// error shape production code actually produces (a *url.Error wrapping the context error, as -// net/http's transport does) instead of a bare context error. -func blockingCallWrappedTimeout(delays []time.Duration) (listActivitiesFunc, *int32) { - var calls int32 - fn := func(ctx context.Context, userKey, applicationName, eventName, startTime, pageToken, filters string, maxResults int64) (*reportsAdmin.Activities, error) { - idx := int(atomic.AddInt32(&calls, 1) - 1) - delay := delays[len(delays)-1] - if idx < len(delays) { - delay = delays[idx] - } - select { - case <-time.After(delay): - return &reportsAdmin.Activities{}, nil - case <-ctx.Done(): - return nil, &url.Error{Op: "Get", URL: "https://example.com", Err: ctx.Err()} - } - } - return fn, &calls -} - func TestRetryListActivities(t *testing.T) { const ( perAttemptTimeout = 20 * time.Millisecond @@ -241,7 +228,9 @@ func TestRetryListActivities(t *testing.T) { t.Run("retries a hung attempt whose timeout arrives wrapped like production's transport error", func(t *testing.T) { // Pins the contract that hungAttempt's errors.Is still finds context.DeadlineExceeded // inside a *url.Error, not just a bare context error. - call, calls := blockingCallWrappedTimeout([]time.Duration{perAttemptTimeout * 3, 0}) + call, calls := blockingCallWithDelaysWrapped([]time.Duration{perAttemptTimeout * 3, 0}, func(err error) error { + return &url.Error{Op: "Get", URL: "https://example.com", Err: err} + }) ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel()