diff --git a/changelog.d/fixed-reader-filtered-catch-up.md b/changelog.d/fixed-reader-filtered-catch-up.md new file mode 100644 index 0000000000..71487619d1 --- /dev/null +++ b/changelog.d/fixed-reader-filtered-catch-up.md @@ -0,0 +1 @@ +- `/api/matrix/lines` reports `more` only when another line the caller may see remains, so clients paging a filtered backlog can stop without an empty follow-up request ([#16530](https://github.com/hivecommons/hive/issues/16530)). diff --git a/src/pkg/dashboard/api_matrix.go b/src/pkg/dashboard/api_matrix.go index be10478ede..684cee8605 100644 --- a/src/pkg/dashboard/api_matrix.go +++ b/src/pkg/dashboard/api_matrix.go @@ -81,6 +81,8 @@ type matrixLinesResponse struct { Seq uint64 `json:"seq"` Source string `json:"source"` Lines []matrixLine `json:"lines"` + // More is true only when another line this caller may see remains. + More bool `json:"more"` // LastIngestAt is when the pass that feeds this source last finished // (RFC 3339), or null before the first; NextIngestInS is the whole // seconds until the next pass is due, 0 when unknown. Together they let @@ -130,7 +132,7 @@ func (s *Server) handleMatrixLines(w http.ResponseWriter, r *http.Request) { } // Filter inside the scan so max counts only lines this caller may see. - lines, latest := src.SinceMatching(since, limit, func(l matrixLine) bool { + lines, latest, more := src.SinceMatching(since, limit, func(l matrixLine) bool { return cfg.RepoAllowed(l.Repo) }) @@ -139,6 +141,7 @@ func (s *Server) handleMatrixLines(w http.ResponseWriter, r *http.Request) { Seq: latest, Source: "pushed-lines", Lines: lines, + More: more, } if last, nextS := src.ingestStatus(src.now()); !last.IsZero() { resp.LastIngestAt = &last diff --git a/src/pkg/dashboard/api_matrix_test.go b/src/pkg/dashboard/api_matrix_test.go index 836635b6cf..f1ff675abe 100644 --- a/src/pkg/dashboard/api_matrix_test.go +++ b/src/pkg/dashboard/api_matrix_test.go @@ -416,6 +416,46 @@ func getMatrixLines(t *testing.T, s *Server, target string) matrixLinesResponse return resp } +// More describes visible work, even when the page is full and newer ring +// entries are all filtered out (#16530). +func TestHandleMatrixLines_MoreCountsOnlyAllowedLines(t *testing.T) { + s := newTestServer() + s.MatrixObserver(config.MatrixConfig{Enabled: true, Repos: []string{"hivecommons/allowed"}}) + src := s.matrix + t.Cleanup(src.close) + helperAddMatrixLine(src, "hivecommons/denied", "hidden before") + helperAddMatrixLine(src, "hivecommons/allowed", "first") + helperAddMatrixLine(src, "hivecommons/denied", "hidden between") + helperAddMatrixLine(src, "hivecommons/allowed", "last") + helperAddMatrixLine(src, "hivecommons/denied", "hidden after") + + for _, tc := range []struct { + name string + query string + seqs []uint64 + more bool + }{ + {"truncated", "since=0&max=1", []uint64{2}, true}, + {"full final page", "since=2&max=1", []uint64{4}, false}, + {"short final page", "since=2&max=2", []uint64{4}, false}, + {"initial backlog", "since=0", []uint64{2, 4}, false}, + {"only hidden lines", "since=4&max=1", nil, false}, + {"caught up", "since=5&max=1", nil, false}, + } { + t.Run(tc.name, func(t *testing.T) { + resp := getMatrixLines(t, s, "/api/matrix/lines?"+tc.query) + if resp.Seq != 5 || resp.More != tc.more || len(resp.Lines) != len(tc.seqs) { + t.Fatalf("response = %+v, want latest=5 more=%v seqs=%v", resp, tc.more, tc.seqs) + } + for i, seq := range tc.seqs { + if resp.Lines[i].Seq != seq { + t.Fatalf("line %d seq = %d, want %d", i, resp.Lines[i].Seq, seq) + } + } + }) + } +} + // /api/matrix/lines reports when the ingest pass last ran and when the next // is due (#16276), null/0 before the first pass, so the kiosk can tell "no // pushes" from "no ingest". diff --git a/src/pkg/dashboard/matrix_source.go b/src/pkg/dashboard/matrix_source.go index 2a891a285c..6891d3fbd2 100644 --- a/src/pkg/dashboard/matrix_source.go +++ b/src/pkg/dashboard/matrix_source.go @@ -383,13 +383,15 @@ func (m *matrixSource) removeLocked(key string) { // Since returns up to limit backlog lines with Seq > seq, oldest first, and the // latest sequence number. limit <= 0 means a default page. func (m *matrixSource) Since(seq uint64, limit int) ([]matrixLine, uint64) { - return m.SinceMatching(seq, limit, nil) + lines, latest, _ := m.SinceMatching(seq, limit, nil) + return lines, latest } // SinceMatching is Since restricted to lines keep accepts (nil keeps all). // limit counts only kept lines, so a page is never emptied by lines the caller // may not see while kept lines remain past it (hivecommons/hive#15979). -func (m *matrixSource) SinceMatching(seq uint64, limit int, keep func(matrixLine) bool) ([]matrixLine, uint64) { +// The final result reports whether another kept line remains after this page. +func (m *matrixSource) SinceMatching(seq uint64, limit int, keep func(matrixLine) bool) ([]matrixLine, uint64, bool) { if limit <= 0 { limit = matrixDefaultSinceMax } @@ -397,13 +399,16 @@ func (m *matrixSource) SinceMatching(seq uint64, limit int, keep func(matrixLine defer m.mu.Unlock() n := len(m.ring) out := make([]matrixLine, 0, min(limit, n)) - for i := 0; i < n && len(out) < limit; i++ { + for i := 0; i < n; i++ { l := m.ring[(m.next+i)%n] if l.Seq > seq && (keep == nil || keep(l)) { + if len(out) == limit { + return out, m.seq, true + } out = append(out, l) } } - return out, m.seq + return out, m.seq, false } // Latest is the sequence number of the newest accepted line; 0 when none.