diff --git a/taggingapi/tag/tag_member_benchmark_test.go b/taggingapi/tag/tag_member_benchmark_test.go index 83f8cb3..a8e24f6 100644 --- a/taggingapi/tag/tag_member_benchmark_test.go +++ b/taggingapi/tag/tag_member_benchmark_test.go @@ -48,7 +48,7 @@ func BenchmarkGenerateBucketedCursor(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { - generateBucketedCursor(i%BucketCount, fmt.Sprintf("member-%d", i), i) + generateBucketedCursor(i%BucketCount, fmt.Sprintf("member-%d", i)) } } @@ -56,13 +56,13 @@ func BenchmarkParseBucketedCursor(b *testing.B) { // Pre-generate cursors cursors := make([]string, 1000) for i := 0; i < 1000; i++ { - cursors[i] = generateBucketedCursor(i%BucketCount, fmt.Sprintf("member-%d", i), i) + cursors[i] = generateBucketedCursor(i%BucketCount, fmt.Sprintf("member-%d", i)) } b.ResetTimer() for i := 0; i < b.N; i++ { - parseBucketedCursor(cursors[i%1000]) + parseBucketedCursor(cursors[i%1000]) //nolint:errcheck // benchmark only measures parse cost } } @@ -202,7 +202,7 @@ func BenchmarkGetMembersV2PaginatedWithCursor(b *testing.B) { } tagId := "benchmark-tag-pagination-cursor" - cursor := generateBucketedCursor(100, "member-12345", 500) + cursor := generateBucketedCursor(100, "member-12345") b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/taggingapi/tag/tag_member_service.go b/taggingapi/tag/tag_member_service.go index 57eb48f..c72eb1d 100644 --- a/taggingapi/tag/tag_member_service.go +++ b/taggingapi/tag/tag_member_service.go @@ -45,12 +45,13 @@ const ( QueryDeleteBucketMetadata = `DELETE FROM tag_member_metadata WHERE tenant_id = ? AND shard_id = ? AND tag_id = ? AND bucket_id = ?` CountMembersCassandraResp = "count" + + InvalidCursorErrorMsg = "invalid pagination cursor" ) type BucketedCursor struct { - BucketId int `json:"bucketId"` - LastMember string `json:"lastMember,omitempty"` - TotalCollected int `json:"totalCollected"` + BucketId int `json:"bucketId"` + LastMember string `json:"lastMember,omitempty"` } type PaginatedMembersResponse struct { @@ -244,9 +245,15 @@ func GetMembersPaginated(tenantId string, tagId string, limit int, cursor string log.Debugf("Getting paginated members for tag %s, limit %d, cursor %s", tagId, limit, cursor) + state, err := parseBucketedCursor(cursor) + if err != nil { + return nil, err + } + populatedBuckets, err := getPopulatedBuckets(tenantId, tagId) if err != nil { log.Errorf("Error getting populated buckets for tag %s: %v", tagId, err) + return nil, fmt.Errorf("failed to get populated buckets: %w", err) } if len(populatedBuckets) == 0 { @@ -255,16 +262,24 @@ func GetMembersPaginated(tenantId string, tagId string, limit int, cursor string log.Debugf("Found %d populated buckets for tag %s", len(populatedBuckets), tagId) - state := parseBucketedCursor(cursor) var allMembers []string - startIndex := 0 + // Find the first populated bucket at or after the cursor position. If none + // exists (buckets were emptied since the previous page), the enumeration is + // complete — do not wrap around to the beginning. + startIndex := -1 for i, bucketId := range populatedBuckets { if bucketId >= state.BucketId { startIndex = i break } } + if startIndex == -1 { + return &PaginatedMembersResponse{ + Data: []string{}, + HasMore: false, + }, nil + } // Build work items for remaining buckets (apply cursor's lastMember to first bucket only) workers := getReadWorkerCount() @@ -285,10 +300,16 @@ func GetMembersPaginated(tenantId string, tagId string, limit int, cursor string orderedResults := fetchBucketsConcurrent(tenantId, tagId, workItems, workers) - // Merge in bucket order, building cursor at the truncation point + // Merge in bucket order, building cursor at the truncation point. + // A failed bucket fails the whole page: returning partial data would let the + // cursor advance past the failed bucket and silently omit its members from + // the enumeration. The caller retries with the same cursor instead. lastProcessedBucketIndex := startIndex - 1 for idx, result := range orderedResults { - if result.err != nil || len(result.members) == 0 { + if result.err != nil { + return nil, fmt.Errorf("failed to fetch members from bucket %d: %w", remainingBuckets[idx], result.err) + } + if len(result.members) == 0 { lastProcessedBucketIndex = startIndex + idx continue } @@ -298,7 +319,7 @@ func GetMembersPaginated(tenantId string, tagId string, limit int, cursor string if len(result.members) > needed { allMembers = append(allMembers, result.members[:needed]...) - nextCursor := generateBucketedCursor(currentBucketId, result.members[needed-1], len(allMembers)) + nextCursor := generateBucketedCursor(currentBucketId, result.members[needed-1]) log.Debugf("Returning %d members for tag %s with more data in bucket %d", len(allMembers), tagId, currentBucketId) return &PaginatedMembersResponse{ @@ -321,7 +342,7 @@ func GetMembersPaginated(tenantId string, tagId string, limit int, cursor string var nextCursor string if hasMore { nextBucketId := populatedBuckets[lastProcessedBucketIndex+1] - nextCursor = generateBucketedCursor(nextBucketId, "", 0) + nextCursor = generateBucketedCursor(nextBucketId, "") } log.Debugf("Returning %d members for tag %s, hasMore: %v", len(allMembers), tagId, hasMore) @@ -360,11 +381,10 @@ func getMembersFromBucket(tenantId string, tagId string, bucketId int, lastMembe } // Cursor management functions -func generateBucketedCursor(bucketId int, lastMember string, totalCollected int) string { +func generateBucketedCursor(bucketId int, lastMember string) string { cursor := BucketedCursor{ - BucketId: bucketId, - LastMember: lastMember, - TotalCollected: totalCollected, + BucketId: bucketId, + LastMember: lastMember, } data, err := json.Marshal(cursor) @@ -375,30 +395,26 @@ func generateBucketedCursor(bucketId int, lastMember string, totalCollected int) return base64.URLEncoding.EncodeToString(data) } -func parseBucketedCursor(cursor string) BucketedCursor { +func parseBucketedCursor(cursor string) (BucketedCursor, error) { if cursor == "" { - return BucketedCursor{BucketId: 0} + return BucketedCursor{BucketId: 0}, nil } data, err := base64.URLEncoding.DecodeString(cursor) if err != nil { - log.Errorf("Error decoding cursor: %v", err) - return BucketedCursor{BucketId: 0} + return BucketedCursor{}, xwcommon.NewRemoteErrorAS(http.StatusBadRequest, InvalidCursorErrorMsg) } var state BucketedCursor if err := json.Unmarshal(data, &state); err != nil { - log.Errorf("Error unmarshaling cursor: %v", err) - return BucketedCursor{BucketId: 0} + return BucketedCursor{}, xwcommon.NewRemoteErrorAS(http.StatusBadRequest, InvalidCursorErrorMsg) } - // Validate cursor values if state.BucketId < 0 || state.BucketId >= BucketCount { - log.Warnf("Invalid bucket ID in cursor: %d, resetting to 0", state.BucketId) - return BucketedCursor{BucketId: 0} + return BucketedCursor{}, xwcommon.NewRemoteErrorAS(http.StatusBadRequest, InvalidCursorErrorMsg) } - return state + return state, nil } func min(a, b int) int { @@ -532,16 +548,22 @@ func fetchMembersFromBucketsConcurrent(tenantId string, tagId string, bucketIds orderedResults := fetchBucketsConcurrent(tenantId, tagId, workItems, workers) - // Merge in bucket order, stop at totalLimit + // Merge in bucket order, stop at totalLimit. + // A failed bucket fails the whole read — a partial result would silently + // under-report the tag's membership. collected := make([]string, 0) - for _, result := range orderedResults { - if result.err != nil || len(result.members) == 0 { - continue - } + for idx, result := range orderedResults { space := totalLimit - len(collected) if space <= 0 { + // Response already full — errors in later buckets are irrelevant return collected, true, nil } + if result.err != nil { + return nil, false, fmt.Errorf("failed to fetch members from bucket %d: %w", bucketIds[idx], result.err) + } + if len(result.members) == 0 { + continue + } if len(result.members) > space { collected = append(collected, result.members[:space]...) return collected, true, nil diff --git a/taggingapi/tag/tag_member_service_bugs_test.go b/taggingapi/tag/tag_member_service_bugs_test.go new file mode 100644 index 0000000..6a966a4 --- /dev/null +++ b/taggingapi/tag/tag_member_service_bugs_test.go @@ -0,0 +1,189 @@ +package tag + +import ( + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/rdkcentral/xconfadmin/common" + + xwcommon "github.com/rdkcentral/xconfwebconfig/common" + "github.com/rdkcentral/xconfwebconfig/db" + + "github.com/gorilla/mux" + "github.com/stretchr/testify/assert" +) + +// mockDbClient injects Cassandra query behavior for service-level tests. +// The embedded interface panics on any method the test does not stub, +// which flags unexpected database usage. +type mockDbClient struct { + db.DatabaseClient + queryFunc func(query string, params ...string) ([]map[string]any, error) +} + +func (m *mockDbClient) QueryXconfDataRows(query string, params ...string) ([]map[string]any, error) { + return m.queryFunc(query, params...) +} + +func withMockDb(t *testing.T, queryFunc func(query string, params ...string) ([]map[string]any, error)) { + t.Helper() + old := db.GetDatabaseClient() + db.SetDatabaseClient(&mockDbClient{queryFunc: queryFunc}) + t.Cleanup(func() { db.SetDatabaseClient(old) }) +} + +func isMetadataQuery(query string) bool { + return strings.Contains(query, "tag_member_metadata") +} + +// BUG-1: a Cassandra failure while listing populated buckets must surface as an +// internal error, not as 404 "tag not found". +func TestGetMembersPaginated_DbErrorIsNot404(t *testing.T) { + setupTestEnvironment() + withMockDb(t, func(query string, params ...string) ([]map[string]any, error) { + return nil, errors.New("cassandra unavailable") + }) + + resp, err := GetMembersPaginated(db.GetDefaultTenantId(), "some-tag", 10, "") + assert.Nil(t, resp) + assert.Error(t, err) + assert.Equal(t, http.StatusInternalServerError, xwcommon.GetXconfErrorStatusCode(err)) +} + +// Unknown tag (no populated buckets, no DB error) is still a 404. +func TestGetMembersPaginated_UnknownTagIs404(t *testing.T) { + setupTestEnvironment() + withMockDb(t, func(query string, params ...string) ([]map[string]any, error) { + return []map[string]any{}, nil + }) + + resp, err := GetMembersPaginated(db.GetDefaultTenantId(), "missing-tag", 10, "") + assert.Nil(t, resp) + assert.Error(t, err) + assert.Equal(t, http.StatusNotFound, xwcommon.GetXconfErrorStatusCode(err)) +} + +// BUG-2: a failed bucket read fails the page instead of silently omitting that +// bucket's members and advancing the cursor past them. +func TestGetMembersPaginated_BucketFetchErrorFailsPage(t *testing.T) { + setupTestEnvironment() + withMockDb(t, func(query string, params ...string) ([]map[string]any, error) { + if isMetadataQuery(query) { + return []map[string]any{{"bucket_id": 1}, {"bucket_id": 2}}, nil + } + if params[2] == "2" { + return nil, errors.New("read timeout") + } + return []map[string]any{{"member": "AA:BB:CC:DD:EE:01"}}, nil + }) + + resp, err := GetMembersPaginated(db.GetDefaultTenantId(), "some-tag", 10, "") + assert.Nil(t, resp) + assert.Error(t, err) + assert.Contains(t, err.Error(), "bucket 2") + assert.Equal(t, http.StatusInternalServerError, xwcommon.GetXconfErrorStatusCode(err)) +} + +// BUG-2 (non-paginated path): GetTagById must not return a silently incomplete +// member list when a bucket read fails. +func TestGetTagById_BucketFetchErrorFails(t *testing.T) { + setupTestEnvironment() + withMockDb(t, func(query string, params ...string) ([]map[string]any, error) { + if isMetadataQuery(query) { + return []map[string]any{{"bucket_id": 7}}, nil + } + return nil, errors.New("read timeout") + }) + + members, truncated, err := GetTagById(db.GetDefaultTenantId(), "some-tag") + assert.Error(t, err) + assert.Nil(t, members) + assert.False(t, truncated) +} + +// BUG-3: a cursor pointing past the last populated bucket ends the enumeration +// instead of wrapping around to bucket 0 and re-delivering everything. +func TestGetMembersPaginated_CursorBeyondLastBucketEndsEnumeration(t *testing.T) { + setupTestEnvironment() + var memberQueries atomic.Int32 + withMockDb(t, func(query string, params ...string) ([]map[string]any, error) { + if isMetadataQuery(query) { + return []map[string]any{{"bucket_id": 1}, {"bucket_id": 2}}, nil + } + memberQueries.Add(1) + return []map[string]any{{"member": "m1"}}, nil + }) + + cursor := generateBucketedCursor(500, "") + resp, err := GetMembersPaginated(db.GetDefaultTenantId(), "some-tag", 10, cursor) + assert.NoError(t, err) + assert.NotNil(t, resp) + assert.Empty(t, resp.Data) + assert.False(t, resp.HasMore) + assert.Empty(t, resp.NextCursor) + assert.Equal(t, int32(0), memberQueries.Load(), "no member fetches expected past the last populated bucket") +} + +// BUG-3 (related): an unparseable cursor is a 400, not a silent restart from +// bucket 0. +func TestGetMembersPaginated_InvalidCursorReturns400(t *testing.T) { + setupTestEnvironment() + withMockDb(t, func(query string, params ...string) ([]map[string]any, error) { + t.Error("no database query expected for an invalid cursor") + return nil, nil + }) + + resp, err := GetMembersPaginated(db.GetDefaultTenantId(), "some-tag", 10, "!!!not-a-cursor!!!") + assert.Nil(t, resp) + assert.Error(t, err) + assert.Equal(t, http.StatusBadRequest, xwcommon.GetXconfErrorStatusCode(err)) +} + +func TestGetTagMembersHandler_InvalidCursorReturns400(t *testing.T) { + setupTestEnvironment() + withMockDb(t, func(query string, params ...string) ([]map[string]any, error) { + return nil, errors.New("unexpected db call") + }) + + req := httptest.NewRequest("GET", "/taggingService/tags/some-tag/members?cursor=%21bad%21", nil) + req = mux.SetURLVars(req, map[string]string{common.Tag: "some-tag"}) + w := httptest.NewRecorder() + GetTagMembersHandler(w, req) + + assert.Equal(t, http.StatusBadRequest, w.Code) + assert.Contains(t, w.Body.String(), InvalidCursorErrorMsg) +} + +// Regression: two-page walk over two buckets returns each member exactly once +// and terminates. +func TestGetMembersPaginated_TwoPageWalk(t *testing.T) { + setupTestEnvironment() + withMockDb(t, func(query string, params ...string) ([]map[string]any, error) { + if isMetadataQuery(query) { + return []map[string]any{{"bucket_id": 1}, {"bucket_id": 2}}, nil + } + switch params[2] { + case "1": + return []map[string]any{{"member": "m1"}}, nil + case "2": + return []map[string]any{{"member": "m2"}}, nil + } + return nil, fmt.Errorf("unexpected bucket %s", params[2]) + }) + + page1, err := GetMembersPaginated(db.GetDefaultTenantId(), "some-tag", 1, "") + assert.NoError(t, err) + assert.Equal(t, []string{"m1"}, page1.Data) + assert.True(t, page1.HasMore) + assert.NotEmpty(t, page1.NextCursor) + + page2, err := GetMembersPaginated(db.GetDefaultTenantId(), "some-tag", 1, page1.NextCursor) + assert.NoError(t, err) + assert.Equal(t, []string{"m2"}, page2.Data) + assert.False(t, page2.HasMore) +} diff --git a/taggingapi/tag/tag_member_service_test.go b/taggingapi/tag/tag_member_service_test.go index 0482bf6..4bdba07 100644 --- a/taggingapi/tag/tag_member_service_test.go +++ b/taggingapi/tag/tag_member_service_test.go @@ -2,8 +2,10 @@ package tag import ( "fmt" + "net/http" "testing" + xwcommon "github.com/rdkcentral/xconfwebconfig/common" "github.com/rdkcentral/xconfwebconfig/db" "github.com/stretchr/testify/assert" ) @@ -43,46 +45,48 @@ func TestGetBucketId(t *testing.T) { func TestParseBucketedCursor(t *testing.T) { // Test empty cursor - cursor := parseBucketedCursor("") + cursor, err := parseBucketedCursor("") + assert.NoError(t, err) assert.Equal(t, 0, cursor.BucketId) assert.Equal(t, "", cursor.LastMember) - assert.Equal(t, 0, cursor.TotalCollected) // Test valid cursor - validCursor := generateBucketedCursor(5, "test-member", 100) - parsed := parseBucketedCursor(validCursor) + validCursor := generateBucketedCursor(5, "test-member") + parsed, err := parseBucketedCursor(validCursor) + assert.NoError(t, err) assert.Equal(t, 5, parsed.BucketId) assert.Equal(t, "test-member", parsed.LastMember) - assert.Equal(t, 100, parsed.TotalCollected) - // Test invalid cursor - invalidCursor := parseBucketedCursor("invalid-cursor") - assert.Equal(t, 0, invalidCursor.BucketId) + // Invalid cursor is a 400 error, not a silent reset to bucket 0 + _, err = parseBucketedCursor("invalid-cursor") + assert.Error(t, err) + assert.Equal(t, http.StatusBadRequest, xwcommon.GetXconfErrorStatusCode(err)) - // Test cursor with invalid bucket ID - invalidBucketCursor := generateBucketedCursor(9999, "member", 100) - parsed2 := parseBucketedCursor(invalidBucketCursor) - assert.Equal(t, 0, parsed2.BucketId, "Invalid bucket ID should be reset to 0") + // Out-of-range bucket ID is a 400 error + invalidBucketCursor := generateBucketedCursor(9999, "member") + _, err = parseBucketedCursor(invalidBucketCursor) + assert.Error(t, err) + assert.Equal(t, http.StatusBadRequest, xwcommon.GetXconfErrorStatusCode(err)) } func TestGenerateBucketedCursor(t *testing.T) { - cursor := generateBucketedCursor(10, "member123", 500) + cursor := generateBucketedCursor(10, "member123") assert.NotEmpty(t, cursor, "Cursor should not be empty") // Should be base64 encoded - parsed := parseBucketedCursor(cursor) + parsed, err := parseBucketedCursor(cursor) + assert.NoError(t, err) assert.Equal(t, 10, parsed.BucketId) assert.Equal(t, "member123", parsed.LastMember) - assert.Equal(t, 500, parsed.TotalCollected) // Test edge cases - cursor2 := generateBucketedCursor(0, "", 0) + cursor2 := generateBucketedCursor(0, "") assert.NotEmpty(t, cursor2, "Cursor should not be empty even with zero values") - parsed2 := parseBucketedCursor(cursor2) + parsed2, err := parseBucketedCursor(cursor2) + assert.NoError(t, err) assert.Equal(t, 0, parsed2.BucketId) assert.Equal(t, "", parsed2.LastMember) - assert.Equal(t, 0, parsed2.TotalCollected) } func TestBucketDistribution(t *testing.T) {