Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions changelog.d/fixed-reader-filtered-catch-up.md
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
- `/api/matrix/lines` reports `more` only when another line the caller may see remains, so clients paging a filtered backlog can stop without an empty follow-up request ([#16530](https://github.com/hivecommons/hive/issues/16530)).
5 changes: 4 additions & 1 deletion src/pkg/dashboard/api_matrix.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,8 @@ type matrixLinesResponse struct {
Seq uint64 `json:"seq"`
Source string `json:"source"`
Lines []matrixLine `json:"lines"`
// More is true only when another line this caller may see remains.
More bool `json:"more"`
// LastIngestAt is when the pass that feeds this source last finished
// (RFC 3339), or null before the first; NextIngestInS is the whole
// seconds until the next pass is due, 0 when unknown. Together they let
Expand Down Expand Up @@ -130,7 +132,7 @@ func (s *Server) handleMatrixLines(w http.ResponseWriter, r *http.Request) {
}

// Filter inside the scan so max counts only lines this caller may see.
lines, latest := src.SinceMatching(since, limit, func(l matrixLine) bool {
lines, latest, more := src.SinceMatching(since, limit, func(l matrixLine) bool {
return cfg.RepoAllowed(l.Repo)
})

Expand All @@ -139,6 +141,7 @@ func (s *Server) handleMatrixLines(w http.ResponseWriter, r *http.Request) {
Seq: latest,
Source: "pushed-lines",
Lines: lines,
More: more,
}
if last, nextS := src.ingestStatus(src.now()); !last.IsZero() {
resp.LastIngestAt = &last
Expand Down
40 changes: 40 additions & 0 deletions src/pkg/dashboard/api_matrix_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -416,6 +416,46 @@ func getMatrixLines(t *testing.T, s *Server, target string) matrixLinesResponse
return resp
}

// More describes visible work, even when the page is full and newer ring
// entries are all filtered out (#16530).
func TestHandleMatrixLines_MoreCountsOnlyAllowedLines(t *testing.T) {
s := newTestServer()
s.MatrixObserver(config.MatrixConfig{Enabled: true, Repos: []string{"hivecommons/allowed"}})
src := s.matrix
t.Cleanup(src.close)
helperAddMatrixLine(src, "hivecommons/denied", "hidden before")
helperAddMatrixLine(src, "hivecommons/allowed", "first")
helperAddMatrixLine(src, "hivecommons/denied", "hidden between")
helperAddMatrixLine(src, "hivecommons/allowed", "last")
helperAddMatrixLine(src, "hivecommons/denied", "hidden after")

for _, tc := range []struct {
name string
query string
seqs []uint64
more bool
}{
{"truncated", "since=0&max=1", []uint64{2}, true},
{"full final page", "since=2&max=1", []uint64{4}, false},
{"short final page", "since=2&max=2", []uint64{4}, false},
{"initial backlog", "since=0", []uint64{2, 4}, false},
{"only hidden lines", "since=4&max=1", nil, false},
{"caught up", "since=5&max=1", nil, false},
} {
t.Run(tc.name, func(t *testing.T) {
resp := getMatrixLines(t, s, "/api/matrix/lines?"+tc.query)
if resp.Seq != 5 || resp.More != tc.more || len(resp.Lines) != len(tc.seqs) {
t.Fatalf("response = %+v, want latest=5 more=%v seqs=%v", resp, tc.more, tc.seqs)
}
for i, seq := range tc.seqs {
if resp.Lines[i].Seq != seq {
t.Fatalf("line %d seq = %d, want %d", i, resp.Lines[i].Seq, seq)
}
}
})
}
}

// /api/matrix/lines reports when the ingest pass last ran and when the next
// is due (#16276), null/0 before the first pass, so the kiosk can tell "no
// pushes" from "no ingest".
Expand Down
13 changes: 9 additions & 4 deletions src/pkg/dashboard/matrix_source.go
Original file line number Diff line number Diff line change
Expand Up @@ -383,27 +383,32 @@ func (m *matrixSource) removeLocked(key string) {
// Since returns up to limit backlog lines with Seq > seq, oldest first, and the
// latest sequence number. limit <= 0 means a default page.
func (m *matrixSource) Since(seq uint64, limit int) ([]matrixLine, uint64) {
return m.SinceMatching(seq, limit, nil)
lines, latest, _ := m.SinceMatching(seq, limit, nil)
return lines, latest
}

// SinceMatching is Since restricted to lines keep accepts (nil keeps all).
// limit counts only kept lines, so a page is never emptied by lines the caller
// may not see while kept lines remain past it (hivecommons/hive#15979).
func (m *matrixSource) SinceMatching(seq uint64, limit int, keep func(matrixLine) bool) ([]matrixLine, uint64) {
// The final result reports whether another kept line remains after this page.
func (m *matrixSource) SinceMatching(seq uint64, limit int, keep func(matrixLine) bool) ([]matrixLine, uint64, bool) {
if limit <= 0 {
limit = matrixDefaultSinceMax
}
m.mu.Lock()
defer m.mu.Unlock()
n := len(m.ring)
out := make([]matrixLine, 0, min(limit, n))
for i := 0; i < n && len(out) < limit; i++ {
for i := 0; i < n; i++ {
l := m.ring[(m.next+i)%n]
if l.Seq > seq && (keep == nil || keep(l)) {
if len(out) == limit {
return out, m.seq, true
}
out = append(out, l)
}
}
return out, m.seq
return out, m.seq, false
}

// Latest is the sequence number of the newest accepted line; 0 when none.
Expand Down
Loading