From 20aa9088c2c7658899eb22993477d297d1520dc8 Mon Sep 17 00:00:00 2001 From: thatguywob Date: Wed, 4 Nov 2026 01:24:52 +0000 Subject: [PATCH 1/6] =?UTF-8?q?=E2=9C=A8=20feat(dashboard):=20feed=20publi?= =?UTF-8?q?shed=20checkpoint=20regions=20to=20the=20reader?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reuse the authorized sentinel observer for complete add/remove regions with whole-region filtering, bounds, continuation and policy epochs. Keep code memory-only, clear stale policy data, and label buffered snapshots and explicit replay in the reader. Hive-Run: hivecommons/hive#16476 Hive-Plan: reader-source-cutover Hive-Spec: code-activity#published-checkpoints Signed-off-by: thatguywob --- .../added-16476-published-checkpoints.md | 1 + src/cmd/hive/main.go | 4 - src/cmd/hive/matrix_ingest.go | 12 +- src/docs/api-reference/dashboard-misc.md | 19 ++ src/docs/dashboard.md | 58 +++- src/pkg/dashboard/api_checkpoint.go | 35 ++ src/pkg/dashboard/api_matrix.go | 5 + src/pkg/dashboard/checkpoint_source.go | 308 ++++++++++++++++++ src/pkg/dashboard/checkpoint_source_test.go | 284 ++++++++++++++++ src/pkg/dashboard/matrix_persist_test.go | 156 ++------- src/pkg/dashboard/matrix_source.go | 217 ++++-------- src/pkg/dashboard/reader_ui_test.go | 91 ++++-- src/pkg/dashboard/server.go | 10 +- src/pkg/dashboard/static-src/js/reader.js | 105 +++--- src/pkg/dashboard/static/index.html | 105 +++--- 15 files changed, 988 insertions(+), 422 deletions(-) create mode 100644 changelog.d/added-16476-published-checkpoints.md create mode 100644 src/pkg/dashboard/api_checkpoint.go create mode 100644 src/pkg/dashboard/checkpoint_source.go create mode 100644 src/pkg/dashboard/checkpoint_source_test.go diff --git a/changelog.d/added-16476-published-checkpoints.md b/changelog.d/added-16476-published-checkpoints.md new file mode 100644 index 0000000000..d3e17c0e17 --- /dev/null +++ b/changelog.d/added-16476-published-checkpoints.md @@ -0,0 +1 @@ +- Feed the Code activity reader whole pushed PR diff regions, including removals, through the existing authorized sentinel observer, with whole-region secret filtering, bounded memory-only history, policy reset, visible freshness, buffered-snapshot labels and explicit replay; stop restoring or writing the legacy Matrix code-history file. diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 7d5be57141..17a2b757cd 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -2464,10 +2464,6 @@ func (b *boot) bootDashboard() { b.bootDashboardWith(defaultBootDashboardDeps()) // persistence enables injected; see bootDashboardDeps. func (b *boot) bootDashboardWith(deps bootDashboardDeps) { b.dashSrv = deps.newServer(b.cfg.Dashboard.Port, b.cfg.Dashboard.AuthToken, b.logger) - // The Matrix backlog survives a restart so the kiosk never blanks - // (#16276); the paired sentinel head map is set in - // applyGitHubClientConfigHooks. - b.dashSrv.SetMatrixPersistPath(matrixLinesStatePath) b.dashSrv.SetIntegrationExecutionEvaluators(b.integrationPreparationEligibility, b.integrationEligibility) worksource.SetRunStageAccessor(b.dashSrv.RunStageAccessor()) b.dashSrv.SetWritebackGate(b.writebackGate()) diff --git a/src/cmd/hive/matrix_ingest.go b/src/cmd/hive/matrix_ingest.go index 18940998db..d9cacb3f93 100644 --- a/src/cmd/hive/matrix_ingest.go +++ b/src/cmd/hive/matrix_ingest.go @@ -20,15 +20,9 @@ import ( // run every couple of minutes on its own goroutine. const matrixIngestInterval = 2 * time.Minute -// matrixLinesStatePath and sentinelHeadsStatePath are where the Matrix -// backlog and the sentinel's evaluated-head map persist on the PVC, each its -// own file like the other /data histories so a parse failure in one never -// takes the other down. Together they let a restarted hive refill the kiosk -// at once and diff only heads that moved while it was down. -const ( - matrixLinesStatePath = "/data/matrix-lines.json" - sentinelHeadsStatePath = "/data/sentinel-heads.json" -) +// sentinelHeadsStatePath keeps the evaluated-head map across restarts. Code +// buffers remain memory-only; restarting never restores captured code. +const sentinelHeadsStatePath = "/data/sentinel-heads.json" // matrixIngestState serializes incremental passes so concurrent callers must // not read the same new head twice. diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 05b5567c16..317eccf99b 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -119,3 +119,22 @@ PR writes. Push and same-head retry observations do not inherit an old fixer, owner or worker observation timestamp. Review completion uses the dispatch ledger's own agent/review-head/decision receipt and aggregate participation; another agent's same-perspective bundle verdict cannot settle that assignment. + +### Published-checkpoint regions + +`GET /api/matrix/lines?format=regions&since=&max=` selects whole +changed regions from the existing authorized sentinel observer. Authentication +and dashboard read permission are required, including in kiosk mode; responses +use `Cache-Control: no-store`. Disabled capture returns 404. An active cold +source returns 200 with an empty `regions` array and null `last_ingest_at`. + +The response contains continuation `seq`, source high-water mark `latest`, policy +`epoch`, source label `Pushed PR`, stage `pushed-checkpoint`, `last_ingest_at`, +`next_ingest_in_s`, and `regions`. Each region has its own sequence, repo, actual +PR number and `pr_url`, head, path, stage, observed `at`, and ordered `lines` +(`op` is `+` or `-`, `line` is untrimmed text). Follow response `seq`, not +`latest`, until caught up. `max` is a line budget (1–500, default 200); the first +eligible region is always returned whole, even if larger than the budget. +Code buffers are bounded and memory-only, so this is not an audit history. +Clear retained browser code on epoch changes and failed/disabled/unauthorized +pulls. An empty response says nothing about work outside the admitted source. diff --git a/src/docs/dashboard.md b/src/docs/dashboard.md index 25a8e220da..0e89e3d839 100644 --- a/src/docs/dashboard.md +++ b/src/docs/dashboard.md @@ -617,31 +617,57 @@ scrub/path filters. When `dashboard.matrix.enabled` is set, the header shows a 404 and the view says it is not enabled. The reader adds no permission and no anonymous access. -- **Cards.** Consecutive lines from one file at one PR head form one card, - which shows that contiguous changed region in order. Indentation, blank lines - and repeated lines are kept. Added lines start with a green `+` and - removed lines (`op: "-"`) with a red `−`. Long lines wrap at a fixed font size and - are never truncated. Every card shows the repository - and PR number, the file, the source stage (`pushed`), and when hive - observed the change. The repository links to the pull request, using the source's - `pr_url` when it is an https URL and otherwise the derived GitHub link. Code is +- **Cards.** Each card is a complete contiguous add/remove region from a + validated unified-diff hunk at an actual published PR head. Indentation, blank lines + and repeated lines are kept. Additions start with a green `+`, and removals + start with a red `−`. Long lines wrap at a + fixed font size and are never truncated. Every card shows the repository + and PR number, the file, the source label (`Pushed PR`), and when hive + observed the change. The repository links to the real pull request. Code is always shown as text and is never parsed as HTML. - **Pacing.** One card is read at a time. A card stays in focus for a few seconds plus a little more for each line, up to 30 seconds, and then the next card scrolls into view. New work waits behind the paragraph being read. - When the reader is caught up, it holds quietly. It has no replay, filler, - typing effect or sound. + When the reader is caught up, it holds quietly. Opening backlog is labelled + **Buffered snapshot**. **Replay buffered snapshot** explicitly replays only + the retained regions with a replay label; automatic replay is off. There is + no filler, typing effect or sound. - **Pause.** Press **P**, press **Space**, or use the **Pause** button to pause or resume. Scrolling with the wheel, touch, arrow keys, Page Up/Page Down, Home or End also pauses the reader so you can inspect the text. The feed is native, scrollable text. - **Motion.** With `prefers-reduced-motion`, every card is shown at once and - nothing scrolls by itself. In a hidden tab, the reader stops advancing until - the tab is visible again. - -The feed currently carries pushed added lines, as shown in Matrix. Removal lines -and whole published-checkpoint regions come from the published-checkpoint -source ([#16476](https://github.com/hivecommons/hive/issues/16476)). + nothing scrolls by itself. Hiding the tab clears buffered browser code; + returning obtains a new buffered snapshot. + +The reader requests `GET /api/matrix/lines?format=regions&since=` through +existing dashboard read authorization. The sentinel's existing optional patch +observer feeds this memory-only projection; no new poller or workspace collector +is installed. Only positively identified Hive/clanker PRs from explicitly +allowlisted, verified-public repositories are eligible. Private or unknown +visibility fails closed. The source reports its last completed ingest pass and +next expected pass; a cold empty feed does **not** mean no work happened. Its +freshness is bounded by sentinel polling, unchanged-head suppression and budgets, +not a live editor stream. + +Changed regions preserve both signs, indentation, blanks and repeated lines. +An unsafe addition **or removal** drops its entire region. Built-in path denies +and configured exclusions still apply. Admission, pagination, deduplication and +eviction operate on whole regions. Limits are 4,096 runes per line, 16 KiB per +region, 200 changed lines per head, 600 per minute, and 2,000 lines / 512 KiB of +server text. Browser history holds at most 1,200 lines / 256 KiB of UTF-16 text +and 32 visible cards, evicting whole regions. These are text bounds, not total +process-memory measurements. Secret filtering remains heuristic. + +Policy changes clear server buffers and rotate a policy epoch. The browser +clears its retained code and replay history on a changed epoch, disabled or +unauthorized response, failed pull, or hidden tab. A 30-second authorized pull +recovers missed sequence-only SSE nudges; code is never sent on SSE or logged. +Buffers no longer persist across restarts, including the legacy Matrix line +projection. Existing `/data/matrix-lines.json` files are neither restored nor +updated; operators may remove such historical files according to their own +retention policy. The sentinel evaluated-head map remains separate and persists. +There is no `dashboard.matrix.workspaces` setting or pre-push capture. ## PR review and link signals diff --git a/src/pkg/dashboard/api_checkpoint.go b/src/pkg/dashboard/api_checkpoint.go new file mode 100644 index 0000000000..8c4a0aa65b --- /dev/null +++ b/src/pkg/dashboard/api_checkpoint.go @@ -0,0 +1,35 @@ +package dashboard + +import ( + "encoding/json" + "net/http" + "strconv" +) + +// handleCheckpointRegions runs behind handleMatrixLines' existing read gate. +// Initializing the memory-only projection here gives an enabled cold source a +// truthful empty 200, without waiting for or initiating a sentinel pass. +func (s *Server) handleCheckpointRegions(w http.ResponseWriter, r *http.Request) { + s.MatrixObserver(s.currentMatrixConfig()) + src, cfg := s.matrixActiveSource() + if src == nil || cfg == nil { + jsonError(w, "matrix mode disabled", http.StatusNotFound) + return + } + since, _ := strconv.ParseUint(r.URL.Query().Get("since"), 10, 64) + limit := matrixDefaultSinceMax + if r.URL.Query().Has("max") { + n, err := strconv.Atoi(r.URL.Query().Get("max")) + if err != nil || n < matrixMinSinceMax { + n = matrixMinSinceMax + } + limit = min(n, matrixMaxSinceMax) + } + resp := src.checkpoints.since(since, limit) + if last, next := src.ingestStatus(src.now()); !last.IsZero() { + resp.LastIngestAt = &last + resp.NextIngestInS = next + } + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(resp) +} diff --git a/src/pkg/dashboard/api_matrix.go b/src/pkg/dashboard/api_matrix.go index b87a39ec18..be10478ede 100644 --- a/src/pkg/dashboard/api_matrix.go +++ b/src/pkg/dashboard/api_matrix.go @@ -97,6 +97,11 @@ func (s *Server) handleMatrixLines(w http.ResponseWriter, r *http.Request) { return } + if r.URL.Query().Get("format") == "regions" { + s.handleCheckpointRegions(w, r) + return + } + src, cfg := s.matrixActiveSource() if src == nil || cfg == nil || !cfg.Active() { jsonError(w, "matrix mode disabled", http.StatusNotFound) diff --git a/src/pkg/dashboard/checkpoint_source.go b/src/pkg/dashboard/checkpoint_source.go new file mode 100644 index 0000000000..a401d790c8 --- /dev/null +++ b/src/pkg/dashboard/checkpoint_source.go @@ -0,0 +1,308 @@ +package dashboard + +import ( + "crypto/rand" + "crypto/sha256" + "encoding/hex" + "fmt" + "regexp" + "slices" + "strconv" + "strings" + "sync" + "time" + "unicode" + "unicode/utf8" + + "github.com/hivecommons/hive/crux/logscrub" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" +) + +const ( + checkpointMaxLineRunes = 4096 + checkpointMaxRegionBytes = 16 << 10 + checkpointMaxBytes = 512 << 10 +) + +type checkpointLine struct { + Op string `json:"op"` + Line string `json:"line"` +} + +type checkpointRegion struct { + Seq uint64 `json:"seq"` + Repo string `json:"repo"` + PR int `json:"pr"` + PRURL string `json:"pr_url"` + Head string `json:"head"` + Path string `json:"path"` + Stage string `json:"stage"` + At time.Time `json:"at"` + Lines []checkpointLine `json:"lines"` +} + +type checkpointHead struct { + at time.Time +} + +// checkpointSource is a memory-only projection of the existing sentinel +// observer. It never fetches patches, restores Matrix's persisted lines, logs +// content, or broadcasts code. A region is the indivisible unit everywhere. +type checkpointSource struct { + mu sync.Mutex + cfg config.MatrixConfig + epoch string + seq uint64 + regions []checkpointRegion + lines, bytes int + heads map[[sha256.Size]byte]checkpointHead + rateStart time.Time + rateCount int +} + +func newCheckpointSource(cfg config.MatrixConfig) *checkpointSource { + c := &checkpointSource{} + c.configure(cfg) + return c +} + +func (c *checkpointSource) configure(cfg config.MatrixConfig) { + c.mu.Lock() + defer c.mu.Unlock() + if c.epoch != "" && c.cfg.Enabled == cfg.Enabled && slices.Equal(c.cfg.Repos, cfg.Repos) && slices.Equal(c.cfg.ExcludePaths, cfg.ExcludePaths) { + return + } + cfg.Repos = append([]string(nil), cfg.Repos...) + cfg.ExcludePaths = append([]string(nil), cfg.ExcludePaths...) + c.cfg = cfg + var nonce [16]byte + _, _ = rand.Read(nonce[:]) // crypto/rand.Read cannot fail on supported Go versions. + c.epoch = hex.EncodeToString(nonce[:]) + c.regions = nil + c.lines, c.bytes, c.rateCount = 0, 0, 0 + c.rateStart = time.Time{} + c.heads = make(map[[sha256.Size]byte]checkpointHead) +} + +func (c *checkpointSource) policyEpoch() string { + c.mu.Lock() + defer c.mu.Unlock() + return c.epoch +} + +func (c *checkpointSource) ingest(obs github.SentinelPatchObservation, epoch string, now time.Time) int { + c.mu.Lock() + defer c.mu.Unlock() + if epoch != c.epoch || !c.cfg.Active() || !c.cfg.RepoAllowed(obs.Repo) || obs.Private || !obs.HiveIdentified || obs.Number <= 0 || obs.HeadSHA == "" { + return 0 + } + // The sentinel proves public visibility (unknown is Private=true). Only its + // actual PR/head observations enter; observing the same head cannot replay it. + key := sha256.Sum256([]byte(strings.ToLower(obs.Repo) + "\x00" + strconv.Itoa(obs.Number) + "\x00" + obs.HeadSHA)) + for k, head := range c.heads { + if now.Sub(head.at) >= matrixDedupTTL { + delete(c.heads, k) + } + } + if _, seen := c.heads[key]; seen { + return 0 + } + if len(c.heads) >= matrixMaxTrackedPRs { + var oldest [sha256.Size]byte + var at time.Time + for k, head := range c.heads { + if at.IsZero() || head.at.Before(at) { + oldest, at = k, head.at + } + } + delete(c.heads, oldest) + } + c.heads[key] = checkpointHead{at: now} + if now.Sub(c.rateStart) >= matrixRateWindow || now.Before(c.rateStart) { + c.rateStart, c.rateCount = now, 0 + } + at := obs.At + if at.IsZero() { + at = now + } + count := 0 + for i, file := range obs.Files { + if i >= matrixMaxFilesPerObs { + break + } + if matrixPathDenied(file.Path, c.cfg.ExcludePaths) || len(file.Patch) > matrixMaxPatchBytes { + continue + } + for _, lines := range checkpointRegions(file.Patch) { + size := 0 + safe := len(lines) <= matrixMaxLinesPerHead + for _, line := range lines { + size += len(line.Line) + safe = safe && checkpointSafeLine(line.Line) + } + if !safe || size > checkpointMaxRegionBytes { + continue + } + if count+len(lines) > matrixMaxLinesPerHead || c.rateCount+len(lines) > matrixMaxLinesPerRate { + continue + } + c.seq++ + region := checkpointRegion{Seq: c.seq, Repo: obs.Repo, PR: obs.Number, PRURL: fmt.Sprintf("https://github.com/%s/pull/%d", obs.Repo, obs.Number), Head: obs.HeadSHA, Path: file.Path, Stage: "pushed-checkpoint", At: at, Lines: lines} + for len(c.regions) > 0 && (c.lines+len(lines) > matrixBacklogCap || c.bytes+size > checkpointMaxBytes) { + old := c.regions[0] + c.lines -= len(old.Lines) + for _, l := range old.Lines { + c.bytes -= len(l.Line) + } + c.regions[0] = checkpointRegion{} + c.regions = c.regions[1:] + } + c.regions = append(c.regions, region) + c.lines += len(lines) + c.bytes += size + count += len(lines) + c.rateCount += len(lines) + } + } + return count +} + +func checkpointSafeLine(line string) bool { + if !utf8.ValidString(line) || utf8.RuneCountInString(line) > checkpointMaxLineRunes { + return false + } + for _, r := range line { + if unicode.IsControl(r) && r != '\t' { + return false + } + } + return !matrixHasEntropyRun(line) && !matrixKeyValueSecret.MatchString(line) && + !strings.Contains(line, "-----BEGIN ") && !strings.Contains(line, "-----END ") && + !logscrub.ContainsRedactionMarker(line) && logscrub.ScrubString(line) == line && logscrub.RedactPaneTokens(line) == line +} + +var checkpointHunk = regexp.MustCompile(`^@@ -[0-9]+(?:,([0-9]+))? \+[0-9]+(?:,([0-9]+))? @@(?:.*)$`) + +// Validate hunk counts before admitting any changed runs. Incomplete or +// malformed patches fail closed, so a truncated hunk never looks complete. +func checkpointRegions(patch string) [][]checkpointLine { + if patch == "" || strings.ContainsRune(patch, 0) { + return nil + } + rows := strings.Split(patch, "\n") + if rows[len(rows)-1] == "" { + rows = rows[:len(rows)-1] + } + var out [][]checkpointLine + var run []checkpointLine + oldLeft, newLeft := 0, 0 + inHunk := false + flush := func() { + if len(run) > 0 { + out = append(out, run) + run = nil + } + } + for _, row := range rows { + if strings.HasPrefix(row, "@@") { + if inHunk && (oldLeft != 0 || newLeft != 0) { + return nil + } + flush() + match := checkpointHunk.FindStringSubmatch(row) + if match == nil { + return nil + } + oldLeft, newLeft = 1, 1 + if match[1] != "" { + n, e := strconv.Atoi(match[1]) + if e != nil { + return nil + } + oldLeft = n + } + if match[2] != "" { + n, e := strconv.Atoi(match[2]) + if e != nil { + return nil + } + newLeft = n + } + inHunk = true + continue + } + if !inHunk { + continue + } + if row == `\ No newline at end of file` { + continue + } + if row == "" { + return nil + } + switch row[0] { + case '+': + newLeft-- + run = append(run, checkpointLine{Op: "+", Line: strings.Clone(row[1:])}) + case '-': + oldLeft-- + run = append(run, checkpointLine{Op: "-", Line: strings.Clone(row[1:])}) + case ' ': + oldLeft-- + newLeft-- + flush() + default: + return nil + } + if oldLeft < 0 || newLeft < 0 { + return nil + } + } + if !inHunk || oldLeft != 0 || newLeft != 0 { + return nil + } + flush() + return out +} + +type checkpointResponse struct { + Seq uint64 `json:"seq"` + Latest uint64 `json:"latest"` + Epoch string `json:"epoch"` + Source string `json:"source"` + Stage string `json:"stage"` + Regions []checkpointRegion `json:"regions"` + LastIngestAt *time.Time `json:"last_ingest_at"` + NextIngestInS int `json:"next_ingest_in_s"` +} + +// max counts changed lines, but the first eligible region is always returned +// whole even when larger than max. seq advances only through consumed regions. +func (c *checkpointSource) since(seq uint64, limit int) checkpointResponse { + c.mu.Lock() + defer c.mu.Unlock() + if limit <= 0 { + limit = matrixDefaultSinceMax + } + if seq > c.seq { + seq = 0 + } + out := checkpointResponse{Seq: seq, Latest: c.seq, Epoch: c.epoch, Source: "Pushed PR", Stage: "pushed-checkpoint", Regions: []checkpointRegion{}} + count := 0 + for _, r := range c.regions { + if r.Seq <= seq { + continue + } + if count > 0 && count+len(r.Lines) > limit { + break + } + out.Regions = append(out.Regions, r) + count += len(r.Lines) + out.Seq = r.Seq + } + if len(out.Regions) == 0 { + out.Seq = c.seq + } + return out +} diff --git a/src/pkg/dashboard/checkpoint_source_test.go b/src/pkg/dashboard/checkpoint_source_test.go new file mode 100644 index 0000000000..58b4df0908 --- /dev/null +++ b/src/pkg/dashboard/checkpoint_source_test.go @@ -0,0 +1,284 @@ +package dashboard + +import ( + "encoding/json" + "fmt" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/sentinel" +) + +func checkpointPatch(lines ...string) string { + return fmt.Sprintf("@@ -0,0 +1,%d @@\n+%s\n", len(lines), strings.Join(lines, "\n+")) +} + +func TestCheckpointWholeRegionsAndContinuation(t *testing.T) { + c := newCheckpointSource(matrixTestConfig()) + patch := "@@ -1,4 +1,7 @@\n-old\n+\tnew\n+\n+repeat\n+repeat\n context\n-again\n+again\n tail\n" + obs := matrixObs(3, "real-head", sentinel.File{Path: "a.go", Patch: patch}) + now := time.Now() + if n := c.ingest(obs, c.policyEpoch(), now); n != 7 { + t.Fatalf("admitted %d changed lines", n) + } + first := c.since(0, 1) + if len(first.Regions) != 1 || len(first.Regions[0].Lines) != 5 || first.Seq != 1 || first.Latest != 2 { + t.Fatalf("first page: %+v", first) + } + want := []checkpointLine{{"-", "old"}, {"+", "\tnew"}, {"+", ""}, {"+", "repeat"}, {"+", "repeat"}} + r := first.Regions[0] + if !reflect.DeepEqual(r.Lines, want) || r.Head != "real-head" || r.PRURL != "https://github.com/"+matrixTestRepo+"/pull/3" || r.Stage != "pushed-checkpoint" || !r.At.Equal(now) { + t.Fatalf("region %+v", r) + } + next := c.since(first.Seq, 1) + if len(next.Regions) != 1 || len(next.Regions[0].Lines) != 2 || next.Seq != 2 || next.Latest != 2 { + t.Fatalf("next page %+v", next) + } + if n := c.ingest(obs, c.policyEpoch(), now); n != 0 { + t.Fatal("unchanged head replayed") + } + obs.HeadSHA = "next-head" + if n := c.ingest(obs, c.policyEpoch(), now); n != 7 { + t.Fatal("new actual head was suppressed") + } +} + +func TestCheckpointDropsUnsafeWholeRegions(t *testing.T) { + bad := []string{"password: hunter2", "token = \"short\"", "-----BEGIN PRIVATE KEY-----", "[REDACTED]", strings.Repeat("a", 33), "bad\x01text", string([]byte{0xff}), strings.Repeat("a ", checkpointMaxLineRunes)} + for _, op := range []string{"+", "-"} { + for i, line := range bad { + t.Run(fmt.Sprintf("%s/%d", op, i), func(t *testing.T) { + c := newCheckpointSource(matrixTestConfig()) + // The secret and innocent line form one region. A separate safe region + // after context survives; no apparently complete fragment of the first does. + oldN, newN := 1, 3 + if op == "-" { + oldN, newN = 3, 1 + } + patch := fmt.Sprintf("@@ -1,%d +1,%d @@\n%ssafe peer\n%s%s\n context\n@@ -10,0 +10 @@\n+visible\n", oldN, newN, op, op, line) + obs := matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: patch}) + c.ingest(obs, c.policyEpoch(), time.Now()) + got := c.since(0, 200) + if len(got.Regions) != 1 || !reflect.DeepEqual(got.Regions[0].Lines, []checkpointLine{{"+", "visible"}}) { + t.Fatalf("unsafe region leaked: %+v", got.Regions) + } + }) + } + } +} + +func TestCheckpointRejectsIncompleteDiff(t *testing.T) { + for _, patch := range []string{"+orphan", "@@ -0,0 +1,2 @@\n+only one\n", "@@ -1 +1 @@\n-old\n", "@@ malformed\n+x\n", "@@ -0,0 +1 @@\n+x\n+y\n", "@@ -0,0 +1 @@\n+x\n@@ -0,0 +2,2 @@\n+truncated\n", "@@ -0,0 +1 @@\n+binary\x00\n"} { + if r := checkpointRegions(patch); len(r) != 0 { + t.Fatalf("incomplete patch admitted: %+v", r) + } + } + r := checkpointRegions("@@ -1 +1 @@\n--- literal\n+++ literal\n\\ No newline at end of file\n") + if len(r) != 1 || len(r[0]) != 2 || r[0][1].Line != "++ literal" { + t.Fatalf("literal diff signs lost: %+v", r) + } +} + +func TestCheckpointObservationAndPolicyBoundaries(t *testing.T) { + for _, kind := range []string{"private-or-unknown", "unidentified", "unlisted", "no-head", "no-pr", "excluded", "secret-path"} { + t.Run(kind, func(t *testing.T) { + cfg := matrixTestConfig() + cfg.ExcludePaths = []string{"skip/**"} + c := newCheckpointSource(cfg) + obs := matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: checkpointPatch("safe")}) + switch kind { + case "private-or-unknown": + obs.Private = true + case "unidentified": + obs.HiveIdentified = false + case "unlisted": + obs.Repo = "other/repo" + case "no-head": + obs.HeadSHA = "" + case "no-pr": + obs.Number = 0 + case "excluded": + obs.Files[0].Path = "skip/a.go" + case "secret-path": + obs.Files[0].Path = ".env" + } + if c.ingest(obs, c.policyEpoch(), time.Now()) != 0 || len(c.since(0, 200).Regions) != 0 { + t.Fatal("boundary admitted content") + } + }) + } + c := newCheckpointSource(matrixTestConfig()) + epoch := c.policyEpoch() + obs := matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: checkpointPatch("safe")}) + c.ingest(obs, epoch, time.Now()) + cfg := matrixTestConfig() + cfg.ExcludePaths = []string{"elsewhere/**"} + c.configure(cfg) + if c.policyEpoch() == epoch || len(c.since(0, 200).Regions) != 0 || c.since(0, 200).Seq != c.since(0, 200).Latest { + t.Fatal("policy change retained code or epoch") + } + if c.ingest(obs, epoch, time.Now()) != 0 { + t.Fatal("old queued observation crossed policy epoch") + } + epoch = c.policyEpoch() + c.configure(cfg) + if c.policyEpoch() != epoch { + t.Fatal("unchanged policy reset epoch") + } + c.ingest(obs, epoch, time.Now()) + cfg.Enabled = false + c.configure(cfg) + if len(c.since(0, 200).Regions) != 0 || c.ingest(obs, c.policyEpoch(), time.Now()) != 0 { + t.Fatal("disable retained or admitted code") + } +} + +func TestCheckpointPolicyClearsSharedQueue(t *testing.T) { + m := newMatrixSource(matrixTestConfig(), nil) + m.Observe(matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: checkpointPatch("queued code")})) + if len(m.in) != 1 { + t.Fatal("observation did not enter queue") + } + cfg := matrixTestConfig() + cfg.Enabled = false + m.setConfig(cfg) + if len(m.in) != 0 { + t.Fatal("policy change retained queued code") + } + m.policyMu.Lock() + done := make(chan struct{}) + go func() { m.Observe(matrixObs(2, "h")); close(done) }() + select { + case <-done: + case <-time.After(time.Second): + m.policyMu.Unlock() + t.Fatal("observer blocked on policy update") + } + m.policyMu.Unlock() + +} + +func TestCheckpointWholeBounds(t *testing.T) { + c := newCheckpointSource(matrixTestConfig()) + now := time.Now() + obs := matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: checkpointPatch(append(matrixUniqueLines("safe", 199), "last", "over")...)}) + if c.ingest(obs, c.policyEpoch(), now) != 0 { + t.Fatal("per-head bound sliced oversized region") + } + for pr := 2; pr < 25; pr++ { + obs = matrixObs(pr, "h", sentinel.File{Path: "a.go", Patch: checkpointPatch(matrixUniqueLines("safe", 99)...)}) + c.ingest(obs, c.policyEpoch(), now.Add(time.Duration(pr)*time.Minute)) + } + got := c.since(0, 10000) + if c.lines > matrixBacklogCap || c.bytes > checkpointMaxBytes || len(got.Regions) != matrixBacklogCap/99 { + t.Fatalf("buffer bounds lines=%d bytes=%d regions=%d", c.lines, c.bytes, len(got.Regions)) + } + for _, r := range got.Regions { + if len(r.Lines) != 99 { + t.Fatal("eviction sliced region") + } + } + // Byte bound is independent of line count, and cloned text keeps retained + // strings from pinning the much larger original patch backing storage. + c = newCheckpointSource(matrixTestConfig()) + text := strings.Repeat("safe text ", 300) + for pr := 1; pr <= 40; pr++ { + c.ingest(matrixObs(pr, "h", sentinel.File{Path: "a.go", Patch: checkpointPatch(text, text, text, text, text)}), c.policyEpoch(), now.Add(time.Duration(pr)*time.Minute)) + } + if c.bytes > checkpointMaxBytes || len(c.regions) >= 40 { + t.Fatal("text byte bound not enforced") + } + c = newCheckpointSource(matrixTestConfig()) + for pr := 1; pr <= 4; pr++ { + n := c.ingest(matrixObs(pr, "h", sentinel.File{Path: "a.go", Patch: checkpointPatch(matrixUniqueLines("safe", 199)...)}), c.policyEpoch(), now) + if (pr <= 3 && n != 199) || (pr == 4 && n != 0) { + t.Fatalf("rate bound split region: pr=%d n=%d", pr, n) + } + } +} + +func TestCheckpointAPIAuthColdAndEpoch(t *testing.T) { + s := NewServerWithAuth(0, "checkpoint-test-token", slog.New(slog.NewTextHandler(io.Discard, nil))) + s.mux.HandleFunc("GET /api/matrix/lines", s.handleMatrixLines) + // Active configuration exists before the sentinel has created its observer. + cfg := matrixTestConfig() + s.deps = &Dependencies{Config: &config.Config{}} + s.deps.Config.Dashboard.Matrix = cfg + defer func() { + if s.matrix != nil { + s.matrix.close() + <-s.matrix.done + } + }() + request := func(auth, role string) *httptest.ResponseRecorder { + req := httptest.NewRequest(http.MethodGet, "/api/matrix/lines?since=0&format=regions&max=1", nil) + req.Header.Set("Authorization", auth) + req.Header.Set("X-Hive-Role", role) + rr := httptest.NewRecorder() + s.Handler().ServeHTTP(rr, req) + if rr.Header().Get("Cache-Control") != "no-store" { + t.Fatal("cacheable response") + } + return rr + } + for _, role := range []string{"", config.RoleOwner} { + if rr := request("", role); rr.Code != 401 { + t.Fatalf("anonymous/forged role got %d", rr.Code) + } + } + rr := request("Bearer checkpoint-test-token", "") + var cold checkpointResponse + if rr.Code != 200 || json.Unmarshal(rr.Body.Bytes(), &cold) != nil || cold.Epoch == "" || cold.LastIngestAt != nil || len(cold.Regions) != 0 || cold.Source != "Pushed PR" || cold.Stage != "pushed-checkpoint" { + t.Fatalf("cold response %d %s", rr.Code, rr.Body.String()) + } + // Feed the real observer queue, including a deletion-only checkpoint. + s.MatrixObserver(cfg)(matrixObs(1, "real-head", sentinel.File{Path: "a.go", Patch: "@@ -1,2 +0,0 @@\n-old\n-old\n"})) + deadline := time.After(2 * time.Second) + for len(s.matrix.checkpoints.since(0, 1).Regions) == 0 { + select { + case <-deadline: + t.Fatal("observer did not feed checkpoint source") + default: + time.Sleep(time.Millisecond) + } + } + s.RecordMatrixIngest(time.Now().UTC(), time.Minute) + rr = request("Bearer checkpoint-test-token", "") + var page checkpointResponse + if json.Unmarshal(rr.Body.Bytes(), &page) != nil || len(page.Regions) != 1 || len(page.Regions[0].Lines) != 2 || page.Seq != page.Latest || page.LastIngestAt == nil { + t.Fatalf("whole response %s", rr.Body.String()) + } + cfg.ExcludePaths = []string{"a.go"} + s.deps.Config.Dashboard.Matrix = cfg + rr = request("Bearer checkpoint-test-token", "") + var changed checkpointResponse + if json.Unmarshal(rr.Body.Bytes(), &changed) != nil || changed.Epoch == cold.Epoch || len(changed.Regions) != 0 { + t.Fatalf("policy change %s", rr.Body.String()) + } + s.deps.Config.Dashboard.Matrix.Enabled = false + if rr := request("Bearer checkpoint-test-token", ""); rr.Code != 404 { + t.Fatalf("disabled=%d", rr.Code) + } +} + +// The legacy persistence format cannot retain or restore checkpoint regions. +func TestCheckpointMemoryOnly(t *testing.T) { + m := newMatrixSource(matrixTestConfig(), nil) + obs := matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: checkpointPatch("safe")}) + m.checkpoints.ingest(obs, m.checkpoints.policyEpoch(), time.Now()) + restored := newMatrixSource(matrixTestConfig(), nil) + if len(restored.checkpoints.since(0, 200).Regions) != 0 || restored.checkpoints.policyEpoch() == m.checkpoints.policyEpoch() { + t.Fatal("new source restored checkpoint history") + } + // Compile-time assertion: the public observer still receives the existing + // authoritative transport, with no extra patch collector or endpoint. + var _ func(github.SentinelPatchObservation) = m.Observe +} diff --git a/src/pkg/dashboard/matrix_persist_test.go b/src/pkg/dashboard/matrix_persist_test.go index fdb4314efc..9489fd28b7 100644 --- a/src/pkg/dashboard/matrix_persist_test.go +++ b/src/pkg/dashboard/matrix_persist_test.go @@ -1,149 +1,45 @@ package dashboard import ( - "encoding/json" "os" "path/filepath" - "strconv" "testing" "time" - "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/sentinel" ) -// The backlog survives a restart (#16276): a new source on the same path -// serves the persisted lines, continues the sequence counter so a kiosk -// never sees seq go backwards, and treats a re-observed head as already -// seen. -func TestMatrixSourcePersistRestore(t *testing.T) { +// The superseded persistent line history must never re-enter either feed. +// Existing files are left for the operator to remove, rather than silently +// deleting their data as part of a dashboard config or software upgrade. +func TestMatrixCodeHistoryIsMemoryOnly(t *testing.T) { path := filepath.Join(t.TempDir(), "matrix-lines.json") - m, clk := newTestMatrixSource(t, matrixTestConfig()) - m.persistPath = path - m.restore() // no file yet is the normal first boot - if m.Latest() != 0 || len(m.ring) != 0 { - t.Fatalf("missing file restored seq=%d lines=%d", m.Latest(), len(m.ring)) - } - file := sentinel.File{Path: "a.go", Patch: matrixPatch("one", "two")} - if got := m.ingest(matrixObs(1, "h1", file)); got != 2 { - t.Fatalf("ingest = %d, want 2", got) - } - m.persist() - raw, err := os.ReadFile(path) - if err != nil { - t.Fatalf("persisted file: %v", err) - } - var f matrixPersistFile - if err := json.Unmarshal(raw, &f); err != nil || f.Seq != 2 || len(f.Lines) != 2 || f.Lines[1].Line != "two" { - t.Fatalf("persisted %+v err=%v", f, err) - } - - restored, _ := newTestMatrixSource(t, matrixTestConfig()) - restored.now = clk.now - restored.persistPath = path - restored.restore() - lines, latest := restored.Since(0, 10) - if latest != 2 || len(lines) != 2 || lines[0].Line != "one" || lines[1].Seq != 2 { - t.Fatalf("restored latest=%d lines=%+v", latest, lines) - } - if got := restored.ingest(matrixObs(1, "h1", file)); got != 0 { - t.Fatalf("re-observed head after restart added %d lines", got) - } - if got := restored.ingest(matrixObs(2, "h", sentinel.File{Path: "b.go", Patch: matrixPatch("three")})); got != 1 { - t.Fatalf("new PR after restart added %d", got) - } - if restored.Latest() != 3 { - t.Fatalf("sequence did not continue: latest=%d", restored.Latest()) - } - - // A source with no path never touches disk. - plain, _ := newTestMatrixSource(t, matrixTestConfig()) - plain.ingest(matrixObs(1, "h1", file)) - plain.persist() - if entries, _ := os.ReadDir(filepath.Dir(path)); len(entries) != 1 { - t.Fatalf("unexpected files: %v", entries) - } -} - -func TestMatrixSourcePersistCapsAndTolerates(t *testing.T) { - dir := t.TempDir() - path := filepath.Join(dir, "matrix-lines.json") - m, _ := newTestMatrixSource(t, matrixTestConfig()) - m.persistPath = path - // More distinct lines than the persist cap, spread over PRs so the - // per-head cap does not bite. - total := matrixPersistLines + 50 - for pr := 0; pr*100 < total; pr++ { - var ls []string - for i := 0; i < 100 && pr*100+i < total; i++ { - ls = append(ls, "line"+strconv.Itoa(pr*100+i)) - } - m.ingest(matrixObs(pr+1, "h", sentinel.File{Path: "a.go", Patch: matrixPatch(ls...)})) - } - if m.Latest() != uint64(total) { - t.Fatalf("latest=%d want %d", m.Latest(), total) - } - m.persist() - restored, _ := newTestMatrixSource(t, matrixTestConfig()) - restored.persistPath = path - restored.restore() - lines, latest := restored.Since(0, matrixBacklogCap) - if latest != uint64(total) || len(lines) != matrixPersistLines || lines[0].Seq != uint64(total-matrixPersistLines+1) { - t.Fatalf("restored latest=%d n=%d first=%d", latest, len(lines), lines[0].Seq) - } - - // Disabling the matrix empties the file too, so a later re-enable does - // not resurrect lines captured under the old config. - m.setConfig(config.MatrixConfig{}) - m.reset() - empty, _ := newTestMatrixSource(t, matrixTestConfig()) - empty.persistPath = path - empty.restore() - if n := len(empty.ring); n != 0 || empty.Latest() != uint64(total) { - t.Fatalf("after reset restored %d lines, seq %d", n, empty.Latest()) - } - - // Garbage on disk is logged and ignored, never fatal. - if err := os.WriteFile(path, []byte("{not json"), 0o644); err != nil { - t.Fatal(err) - } - bad, _ := newTestMatrixSource(t, matrixTestConfig()) - bad.persistPath = path - bad.restore() - if bad.Latest() != 0 || len(bad.ring) != 0 { - t.Fatalf("malformed file restored seq=%d lines=%d", bad.Latest(), len(bad.ring)) - } -} - -// Server.SetMatrixPersistPath hands the path to the source MatrixObserver -// builds, and a source built first still picks the backlog up when it has -// nothing of its own. -func TestServerSetMatrixPersistPath(t *testing.T) { - path := filepath.Join(t.TempDir(), "matrix-lines.json") - seed := matrixPersistFile{Seq: 7, Lines: []matrixLine{{Seq: 7, Repo: matrixTestRepo, PR: 1, Head: "h", Path: "a.go", Line: "kept", At: time.Now().UTC()}}} - raw, _ := json.Marshal(seed) - if err := os.WriteFile(path, raw, 0o644); err != nil { + seed := []byte(`{"seq":7,"lines":[{"seq":7,"repo":"org/repo","pr":1,"line":"old retained code"}]}`) + if err := os.WriteFile(path, seed, 0600); err != nil { t.Fatal(err) } s := &Server{} s.SetMatrixPersistPath(path) - if s.MatrixObserver(matrixTestConfig()) == nil { - t.Fatal("no observer") - } - t.Cleanup(s.matrix.close) - if lines, latest := s.matrix.Since(0, 10); latest != 7 || len(lines) != 1 || lines[0].Line != "kept" { - t.Fatalf("observer-built source: latest=%d lines=%+v", latest, lines) - } - - late := &Server{} - if late.MatrixObserver(matrixTestConfig()) == nil { - t.Fatal("no observer") + observe := s.MatrixObserver(matrixTestConfig()) + defer func() { s.matrix.close(); <-s.matrix.done }() + if s.matrix.Latest() != 0 || len(s.matrix.checkpoints.since(0, 200).Regions) != 0 { + t.Fatal("legacy persistent code restored") + } + observe(matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: checkpointPatch("fresh checkpoint")})) + deadline := time.After(2 * time.Second) + for s.matrix.Latest() == 0 { + select { + case <-deadline: + t.Fatal("observer did not ingest") + default: + time.Sleep(time.Millisecond) + } } - t.Cleanup(late.matrix.close) - late.SetMatrixPersistPath(path) - if latest := late.matrix.Latest(); latest != 7 { - t.Fatalf("late path on an empty source: latest=%d, want 7", latest) + s.SetMatrixPersistPath(path) + got, err := os.ReadFile(path) + if err != nil || string(got) != string(seed) { + t.Fatal("legacy history file changed") } - var nilSrv *Server - nilSrv.SetMatrixPersistPath(path) + var nilServer *Server + nilServer.SetMatrixPersistPath(path) } diff --git a/src/pkg/dashboard/matrix_source.go b/src/pkg/dashboard/matrix_source.go index 6174bd5e40..92b2c709cb 100644 --- a/src/pkg/dashboard/matrix_source.go +++ b/src/pkg/dashboard/matrix_source.go @@ -2,11 +2,7 @@ package dashboard import ( "crypto/sha256" - "encoding/json" - "errors" - "fmt" "log/slog" - "os" "strconv" "sync" "sync/atomic" @@ -30,20 +26,8 @@ const ( matrixMaxFilesPerObs = 300 matrixMaxPatchBytes = 512 << 10 matrixDefaultSinceMax = 200 - // matrixPersistLines is how many of the newest lines survive a restart - // (#16276): enough to refill the kiosk's 120-line window with headroom, - // small enough that each write is a few tens of kilobytes. - matrixPersistLines = 500 ) -// matrixPersistFile is the on-disk shape of the backlog. Seq is the newest -// sequence number so the counter continues across restarts and a kiosk -// never sees seq go backwards (matrix.js resets on that). -type matrixPersistFile struct { - Seq uint64 `json:"seq"` - Lines []matrixLine `json:"lines"` -} - // matrixMaxDedupHashes caps dedup hashes across every tracked PR; a var so // tests can exercise the bound without 100k lines. var matrixMaxDedupHashes = 100000 @@ -70,12 +54,21 @@ type matrixPRState struct { // matrixSource turns sentinel patch observations into a bounded, scrubbed, // de-duplicated in-memory backlog. Observe never blocks; one goroutine does // all the parsing. Nothing is written to disk. +type matrixQueuedObservation struct { + observation github.SentinelPatchObservation + epoch string +} + +// matrixSource owns the shared observer queue and its legacy line projection. type matrixSource struct { - cfg atomic.Pointer[config.MatrixConfig] - in chan github.SentinelPatchObservation - dropped atomic.Uint64 - logger *slog.Logger - now func() time.Time + checkpoints *checkpointSource + policyMu sync.Mutex + queuedEpoch atomic.Pointer[string] + cfg atomic.Pointer[config.MatrixConfig] + in chan matrixQueuedObservation + dropped atomic.Uint64 + logger *slog.Logger + now func() time.Time startOnce sync.Once stopOnce sync.Once @@ -99,11 +92,6 @@ type matrixSource struct { // /api/matrix/lines so the kiosk can tell "no pushes" from "no ingest". ingestAt atomic.Int64 ingestEvery atomic.Int64 - - // persistPath, when set, receives the newest matrixPersistLines lines - // and the sequence counter after every ingest that added lines and on - // reset; restore reads it back on construction. - persistPath string } func newMatrixSource(cfg config.MatrixConfig, logger *slog.Logger) *matrixSource { @@ -111,7 +99,7 @@ func newMatrixSource(cfg config.MatrixConfig, logger *slog.Logger) *matrixSource logger = slog.Default() } m := &matrixSource{ - in: make(chan github.SentinelPatchObservation, matrixIngestCap), + in: make(chan matrixQueuedObservation, matrixIngestCap), logger: logger, now: time.Now, stop: make(chan struct{}), @@ -119,15 +107,35 @@ func newMatrixSource(cfg config.MatrixConfig, logger *slog.Logger) *matrixSource ring: make([]matrixLine, 0, matrixBacklogCap), prs: make(map[string]*matrixPRState), } + m.checkpoints = newCheckpointSource(cfg) m.setConfig(cfg) return m } func (m *matrixSource) setConfig(cfg config.MatrixConfig) { + m.policyMu.Lock() + defer m.policyMu.Unlock() c := cfg c.Repos = append([]string(nil), cfg.Repos...) c.ExcludePaths = append([]string(nil), cfg.ExcludePaths...) m.cfg.Store(&c) + if m.checkpoints != nil { + epoch := m.checkpoints.policyEpoch() + m.checkpoints.configure(c) + nextEpoch := m.checkpoints.policyEpoch() + m.queuedEpoch.Store(&nextEpoch) + if epoch != nextEpoch { + m.reset() + // Drop queued code as well as admitted code on policy changes. + for { + select { + case <-m.in: + default: + return + } + } + } + } } // admits applies the observation-level gates: Matrix active, hive-identified @@ -141,11 +149,16 @@ func (m *matrixSource) admits(obs github.SentinelPatchObservation) bool { // it off without blocking: when the ingest channel is full the observation // is dropped and counted. func (m *matrixSource) Observe(obs github.SentinelPatchObservation) { + if !m.policyMu.TryLock() { + m.dropped.Add(1) + return + } + defer m.policyMu.Unlock() if !m.admits(obs) { return } select { - case m.in <- obs: + case m.in <- matrixQueuedObservation{observation: obs, epoch: *m.queuedEpoch.Load()}: default: m.dropped.Add(1) } @@ -168,112 +181,44 @@ func (m *matrixSource) run() { select { case <-m.stop: return - case obs := <-m.in: - if m.safeIngest(obs) > 0 { - m.persist() + case queued := <-m.in: + if queued.epoch != m.checkpoints.policyEpoch() { + continue + } + before := m.Latest() + added := m.checkpoints.ingest(queued.observation, queued.epoch, m.now()) + m.safeIngest(queued.observation) + if added > 0 && m.Latest() == before { + // Deletion-only checkpoints also wake the reader. The frame + // contains only the legacy sequence hint, never code. + m.nudgeCheckpoint() } } } } -func (m *matrixSource) safeIngest(obs github.SentinelPatchObservation) (added int) { +func (m *matrixSource) nudgeCheckpoint() { defer func() { - if r := recover(); r != nil { - added = 0 - m.logger.Warn("matrix line source dropped an observation after a panic", "repo", obs.Repo, "number", obs.Number, "panic", fmt.Sprint(r)) + if recover() != nil { + m.logger.Warn("checkpoint notification failed") } }() - return m.ingest(obs) -} - -// restore loads the persisted backlog: the lines refill the ring oldest -// first, the sequence counter resumes from the file, and each line's hash -// re-enters its PR's dedup set so re-observing the same head after a -// restart adds nothing. A missing file is the normal first boot; a -// malformed one is logged and ignored. -func (m *matrixSource) restore() { - if m.persistPath == "" { - return - } - raw, err := os.ReadFile(m.persistPath) - if err != nil { - if !errors.Is(err, os.ErrNotExist) { - m.logger.Warn("matrix line source: state file unreadable; starting empty", "path", m.persistPath, "error", err) - } - return - } - var f matrixPersistFile - if err := json.Unmarshal(raw, &f); err != nil { - m.logger.Warn("matrix line source: state file malformed; starting empty", "path", m.persistPath, "error", err) - return - } - if len(f.Lines) > matrixPersistLines { - f.Lines = f.Lines[len(f.Lines)-matrixPersistLines:] - } - now := m.now() m.mu.Lock() - defer m.mu.Unlock() - var last uint64 - for _, l := range f.Lines { - if l.Line == "" || l.Seq <= last { - continue - } - last = l.Seq - m.appendLocked(l) - key := l.Repo + "#" + strconv.Itoa(l.PR) - st := m.prs[key] - if st == nil { - st = &matrixPRState{hashes: make(map[[sha256.Size]byte]time.Time)} - m.prs[key] = st - } - st.touched = now - if st.head != l.Head { - st.head = l.Head - st.headCount = 0 - } - h := sha256.Sum256([]byte(l.Path + "\x00" + l.Line)) - if _, dup := st.hashes[h]; !dup { - m.totalHashes++ - } - st.hashes[h] = l.At - st.headCount++ - } - m.seq = max(f.Seq, last) - if m.seq > 0 { - m.logger.Info("matrix line source restored", "lines", len(m.ring), "seq", m.seq) + seq, notify := m.seq, m.notify + m.mu.Unlock() + if notify != nil { + notify(seq) } } -// persist writes the newest matrixPersistLines lines and the sequence -// counter to persistPath atomically. It snapshots under the lock and writes -// outside it; no path is a no-op. -func (m *matrixSource) persist() { - if m.persistPath == "" { - return - } - m.mu.Lock() - n := len(m.ring) - start := 0 - if n > matrixPersistLines { - start = n - matrixPersistLines - } - f := matrixPersistFile{Seq: m.seq, Lines: make([]matrixLine, 0, n-start)} - for i := start; i < n; i++ { - f.Lines = append(f.Lines, m.ring[(m.next+i)%n]) - } - m.mu.Unlock() - raw, err := json.Marshal(f) - if err != nil { - return - } - tmp := m.persistPath + ".tmp" - err = os.WriteFile(tmp, raw, 0o644) - if err == nil { - err = os.Rename(tmp, m.persistPath) - } - if err != nil { - m.logger.Warn("matrix line source: state file write failed", "path", m.persistPath, "error", err) - } +func (m *matrixSource) safeIngest(obs github.SentinelPatchObservation) (added int) { + defer func() { + if r := recover(); r != nil { + added = 0 + m.logger.Warn("matrix line source dropped an observation after a panic", "repo", obs.Repo, "number", obs.Number) + } + }() + return m.ingest(obs) } // ingest filters one observation into the backlog and returns how many lines @@ -446,12 +391,12 @@ func (m *matrixSource) Latest() uint64 { // never sees a sequence number reused. func (m *matrixSource) reset() { m.mu.Lock() + clear(m.ring) m.ring = m.ring[:0] m.next = 0 m.prs = make(map[string]*matrixPRState) m.totalHashes = 0 m.mu.Unlock() - m.persist() } // recordIngest notes that an ingest pass finished at `at` and the next one @@ -505,8 +450,6 @@ func (s *Server) MatrixObserver(cfg config.MatrixConfig) func(github.SentinelPat } if s.matrix == nil { s.matrix = newMatrixSource(cfg, s.logger) - s.matrix.persistPath = s.matrixPersistPath - s.matrix.restore() s.matrix.setNotify(s.nudgeMatrixLines) s.matrix.start() } else { @@ -533,23 +476,7 @@ func (s *Server) RecordMatrixIngest(at time.Time, every time.Duration) { m.recordIngest(at, every) } -// SetMatrixPersistPath names the file the Matrix line source keeps its -// newest lines and sequence counter in (#16276), so a restart refills the -// kiosk instead of showing it empty until the next sweep. It must be called -// before the first active MatrixObserver; empty keeps the source in-memory. -func (s *Server) SetMatrixPersistPath(path string) { - if s == nil { - return - } - s.matrixMu.Lock() - defer s.matrixMu.Unlock() - s.matrixPersistPath = path - // A source that already exists but has nothing yet can still take the - // backlog; one with lines keeps them and only persists from here on. - if s.matrix != nil && s.matrix.persistPath == "" && path != "" { - s.matrix.persistPath = path - if s.matrix.Latest() == 0 { - s.matrix.restore() - } - } -} +// SetMatrixPersistPath is retained for callers compiled against the Matrix +// prototype. Code activity is memory-only: no legacy file is read or written. +// Deprecated: persistent code history is no longer supported. +func (s *Server) SetMatrixPersistPath(_ string) {} diff --git a/src/pkg/dashboard/reader_ui_test.go b/src/pkg/dashboard/reader_ui_test.go index 88b566c977..2a7d8a3acb 100644 --- a/src/pkg/dashboard/reader_ui_test.go +++ b/src/pkg/dashboard/reader_ui_test.go @@ -23,7 +23,7 @@ func TestReaderUIWiring(t *testing.T) { t.Errorf("reader.js contains forbidden sink/effect %q", forbidden) } } - for _, want := range []string{"visibilitychange", "prefers-reduced-motion: reduce", "'/api/matrix/lines?since='", "source.addEventListener('code-lines'", "'wheel'", "'touchstart'"} { + for _, want := range []string{"visibilitychange", "prefers-reduced-motion: reduce", "'&format=regions'", "source.addEventListener('code-lines'", "'wheel'", "'touchstart'", "'\\u2212 '"} { if !strings.Contains(js, want) { t.Errorf("reader.js missing %q", want) } @@ -103,12 +103,11 @@ const first = [ {seq: 3, repo: 'org/repo', pr: 2, head: 'abc', path: 'main.go', line: '\tx := 1', at}, {seq: 4, repo: 'org/repo', pr: 2, head: 'abc', path: 'main.go', line: '\tx := 1', at}, {seq: 5, repo: 'org/repo', pr: 2, head: 'abc', path: 'main.go', line: ' old()', op: '-', at}, - {seq: 6, repo: 'org/other', pr: 7, head: 'def', path: 'README.md', line: 'a very long line '.repeat(40), pr_url: 'https://example.test/review/7', at}, + {seq: 6, repo: 'org/other', pr: 7, head: 'def', path: 'README.md', line: 'a very long line '.repeat(40), pr_url: 'https://github.com/org/other/pull/7', at}, ]; async function scenario(query, opts = {}) { const docEvents = {}, timers = new Map(), requests = [], scrolls = [], sse = {}; - // page caps each response like the server does; seq always reports the newest line held. - let id = 0, status = opts.status || 200, seq = 6, lines = first, page = opts.page || Infinity; + let id = 0, status = opts.status || 200, seq = 6, lines = first; const elements = []; function el(tag) { const e = {tagName: tag.toUpperCase(), children: [], listeners: {}, attrs: {}, className: '', textContent: '', hidden: false, scrollTop: 0, offsetHeight: 10, classList: {add(){}}, @@ -125,52 +124,84 @@ async function scenario(query, opts = {}) { localStorage: {getItem(){return null}, setItem(){}}, requestAnimationFrame(){throw new Error('reader must not animate frames')}, setTimeout(fn){timers.set(++id, fn); return id}, clearTimeout(i){timers.delete(i)}, setInterval(){}, dashboardSetInterval(){}, - fetch(url){requests.push(url); const body = {seq, source: 'pushed-lines', lines: lines.slice(0, page)}; lines = lines.slice(page); + fetch(url){requests.push(url); const regions = []; + lines.forEach(line => { + let last = regions[regions.length - 1]; + if (!last || last.path !== line.path || last.repo !== line.repo) regions.push(last = {...line, seq: line.seq, lines: []}); + last.lines.push({op: line.op || '+', line: line.line}); + }); + let body = {seq, latest: seq, epoch: 'policy-1', source: 'Pushed PR', regions}; lines = []; + if (opts.pages && opts.pages.length) body = opts.pages.shift(); return Promise.resolve({status, ok: status === 200, json(){return Promise.resolve(body)}})}, Date, Math}); vm.runInContext(reader, c); c.hiveReaderStart = win.hiveReaderStart; vm.runInContext(boot, c); for (const fn of docEvents.DOMContentLoaded || []) fn(); - const flush = async () => {for (let i = 0; i < 12; i++) await Promise.resolve()}; + const flush = async () => {for (let i = 0; i < 64; i++) await Promise.resolve()}; await flush(); if (!win.HIVE_READER) {assert.equal(requests.length, 0); assert(!doc.getElementById('hive-reader')); return;} const byClass = n => elements.find(e => e.className === n); const feed = byClass('reader-feed'), statusEl = byClass('reader-status'), button = byClass('reader-pause'); const fire = () => {assert.equal(timers.size, 1); const [k, fn] = [...timers][0]; timers.delete(k); fn()}; const key = (key, code) => docEvents.keydown[0]({key, code, target: doc.body, preventDefault(){}}); - assert.equal(requests[0], '/api/matrix/lines?since=0'); + assert.equal(requests[0], '/api/matrix/lines?since=0&format=regions'); if (status !== 200) { assert.equal(feed.children.length, 0); assert.equal(timers.size, 0); assert(statusEl.textContent.includes(status === 404 ? 'not enabled' : 'Sign in')); return; } const meta = card => card.children[0].children, rows = card => card.children[1].children.map(r => r.textContent); + if (opts.boundaries) { + // Two complete regions in one file remain separate across pages. The + // reader follows continuation rather than skipping to source.latest. + assert.equal(requests.length, 2); + assert.equal(requests[1], '/api/matrix/lines?since=1&format=regions'); + assert.equal(feed.children.length, 2); + assert.deepEqual(rows(feed.children[0]), ['+ one\n']); + assert.deepEqual(rows(feed.children[1]), ['− two\n']); + assert.equal(meta(feed.children[1])[3].textContent, 'Buffered snapshot'); + win.hiveReaderAttach({addEventListener(n, fn){sse[n] = fn}}); + opts.pages.push({seq: 3, latest: 3, epoch: 'policy-2', regions: [{repo: 'org/repo', pr: 2, path: 'main.go', seq: 3, at, lines: [{op: '+', line: 'new policy'}]}]}); + sse['code-lines'](); await flush(); + assert.equal(feed.children.length, 1); + assert.deepEqual(rows(feed.children[0]), ['+ new policy\n']); + byClass('reader-pause reader-replay').listeners.click(); + assert.equal(feed.children.length, 1); // old policy is gone from replay too + assert.deepEqual(rows(feed.children[0]), ['+ new policy\n']); + // Busy feeds evict whole regions under line, text and viewport bounds. + const base = {repo: 'org/repo', pr: 2, path: 'main.go', at}; + opts.pages.push({seq: 10, latest: 10, epoch: 'policy-2', regions: Array.from({length: 7}, (_, i) => ({...base, seq: i + 4, lines: Array.from({length: 200}, () => ({op: '+', line: 'bounded line'}))}))}); + sse['code-lines'](); await flush(); + assert(feed.children.reduce((n, card) => n + rows(card).length, 0) <= 1200); + assert(feed.children.every(card => rows(card).length === 1 || rows(card).length === 200)); + opts.pages.push({seq: 50, latest: 50, epoch: 'policy-2', regions: Array.from({length: 40}, (_, i) => ({...base, seq: i + 11, lines: [{op: '+', line: 'one'}]}))}); + sse['code-lines'](); await flush(); assert.equal(feed.children.length, 32); + opts.pages.push({seq: 70, latest: 70, epoch: 'policy-2', regions: Array.from({length: 20}, (_, i) => ({...base, seq: i + 51, lines: [{op: '+', line: 'safe '.repeat(3000)}]}))}); + sse['code-lines'](); await flush(); + assert(feed.children.reduce((n, card) => n + rows(card).join('').length * 2, 0) < 256 * 1024); + status = 404; sse['code-lines'](); await flush(); + assert.equal(feed.children.length, 0); + return; + } + const checkFirst = card => { // One contiguous region: blank, indented and repeated lines kept as text. assert.deepEqual(rows(card), ['+ \n', '+ \n', '+ \tx := 1\n', '+ \tx := 1\n', '\u2212 old()\n']); assert.equal(card.children[1].tagName, 'PRE'); assert.deepEqual(card.children[1].children.map(r => r.className), ['reader-line reader-add', 'reader-line reader-add', 'reader-line reader-add', 'reader-line reader-add', 'reader-line reader-del']); - const [repo, file, stage, time] = meta(card); + const [repo, file, stage, delivery, time] = meta(card); assert.equal(repo.tagName, 'A'); assert.equal(repo.textContent, 'org/repo #2'); assert.equal(repo.href, 'https://github.com/org/repo/pull/2'); assert.equal(repo.attrs.rel, 'noopener noreferrer'); - assert.equal(file.textContent, 'main.go'); assert.equal(stage.textContent, 'pushed'); + assert.equal(file.textContent, 'main.go'); assert.equal(stage.textContent, 'Pushed PR'); assert.equal(delivery.textContent, 'Buffered snapshot'); assert.equal(time.tagName, 'TIME'); assert.equal(time.attrs.datetime, '2026-11-03T12:00:00.000Z'); assert(time.textContent.startsWith('observed ')); }; const checkSecond = card => { - assert.equal(meta(card)[0].href, 'https://example.test/review/7'); + assert.equal(meta(card)[0].href, 'https://github.com/org/other/pull/7'); assert.equal(meta(card)[1].textContent, 'README.md'); assert.equal(rows(card)[0], '+ ' + 'a very long line '.repeat(40) + '\n'); }; win.hiveReaderAttach({addEventListener(n, fn){sse[n] = fn}}); - if (opts.page) { - // A capped page resumes from the last returned line, not the newest seq, - // so no line is skipped and the region stays one contiguous card. - await flush(); - assert.deepEqual(requests, ['/api/matrix/lines?since=0', '/api/matrix/lines?since=2', '/api/matrix/lines?since=4']); - assert.equal(feed.children.length, 2); checkFirst(feed.children[0]); checkSecond(feed.children[1]); - return; - } if (opts.reduced) { // Static and scrollable: everything shown at once, nothing scheduled or scrolled. assert.equal(feed.children.length, 2); assert.equal(timers.size, 0); assert.equal(scrolls.length, 0); @@ -187,7 +218,7 @@ async function scenario(query, opts = {}) { // Incoming work queues behind the paragraph being read. lines = [{seq: 7, repo: 'org/repo', pr: 2, head: 'abc', path: 'b.go', line: 'next', at}]; seq = 7; sse['code-lines'](); await flush(); - assert.equal(requests.at(-1), '/api/matrix/lines?since=6'); + assert.equal(requests.at(-1), '/api/matrix/lines?since=6&format=regions'); assert.equal(feed.children.length, 1); assert.equal(timers.size, 1); // P pauses and resumes; the button mirrors it. key('p'); assert.equal(timers.size, 0); assert.equal(button.textContent, 'Resume'); @@ -200,14 +231,18 @@ async function scenario(query, opts = {}) { byClass('reader-scroll').listeners.touchstart(); assert.equal(timers.size, 0); button.listeners.click(); key('PageDown'); assert.equal(timers.size, 0); button.listeners.click(); assert.equal(timers.size, 1); - // A hidden tab stops all motion. - doc.hidden = true; docEvents.visibilitychange.forEach(fn => fn()); assert.equal(timers.size, 0); - doc.hidden = false; docEvents.visibilitychange.forEach(fn => fn()); await flush(); + // Quiet hold after consuming the queued paragraph; no automatic replay. fire(); assert.equal(feed.children.length, 3); assert.equal(rows(feed.children[2])[0], '+ next\n'); - // Caught up: a quiet hold with no replay or filler. - fire(); assert.equal(timers.size, 0); assert.equal(feed.children.length, 3); - assert(statusEl.textContent.includes('Caught up')); + fire(); assert.equal(timers.size, 0); assert(statusEl.textContent.includes('Caught up')); sse['code-lines'](); await flush(); assert.equal(timers.size, 0); assert.equal(feed.children.length, 3); + // Replay requires a deliberate click and is labelled on every card. + byClass('reader-pause reader-replay').listeners.click(); + assert.equal(feed.children.length, 1); assert.equal(meta(feed.children[0])[3].textContent, 'Replay · buffered snapshot'); + doc.hidden = true; docEvents.visibilitychange.forEach(fn => fn()); + assert.equal(timers.size, 0); assert.equal(feed.children.length, 0); + doc.hidden = false; docEvents.visibilitychange.forEach(fn => fn()); await flush(); + assert.equal(feed.children.length, 0); assert(statusEl.textContent.includes('No buffered checkpoints')); + assert(byClass('reader-freshness').textContent.includes('empty does not mean no work')); status = 403; sse['code-lines'](); await flush(); assert.equal(feed.children.length, 0); assert(statusEl.textContent.includes('Sign in')); } @@ -215,7 +250,11 @@ async function scenario(query, opts = {}) { await scenario(''); await scenario('?view=reader'); await scenario('?kiosk=1'); await scenario('?kiosk=1&view=matrix'); await scenario('?kiosk=1&view=reader'); await scenario('?kiosk=1&view=reader', {reduced: true}); - await scenario('?kiosk=1&view=reader', {reduced: true, page: 2}); + const region = {repo: 'org/repo', pr: 2, path: 'main.go', at}; + await scenario('?kiosk=1&view=reader', {reduced: true, boundaries: true, pages: [ + {seq: 1, latest: 2, epoch: 'policy-1', regions: [{...region, seq: 1, lines: [{op: '+', line: 'one'}]}]}, + {seq: 2, latest: 2, epoch: 'policy-1', regions: [{...region, seq: 2, lines: [{op: '-', line: 'two'}]}]}, + ]}); for (const status of [404, 401, 403]) await scenario('?kiosk=1&view=reader', {status}); })().catch(e => {console.error(e); process.exit(1)}); ` diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 78b8f15f1f..62a6fc0255 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -119,10 +119,9 @@ type Server struct { // matrix is the dashboard.matrix line source (matrix_source.go); nil // until MatrixObserver is called with an active config. - matrixMu sync.Mutex - matrix *matrixSource - matrixNudge stageTrackerNudger - matrixPersistPath string + matrixMu sync.Mutex + matrix *matrixSource + matrixNudge stageTrackerNudger port int authToken string @@ -1979,6 +1978,9 @@ code{color:#e6edf3} // empty role and never block a read-only viewer's writes.) func (s *Server) authenticate(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/matrix/lines" { + w.Header().Set("Cache-Control", "no-store") + } directRouteAuthz := s.directRouteAuthzEnabled() // FAIL CLOSED: a spoke with an authorized_users allowlist (direct-route) diff --git a/src/pkg/dashboard/static-src/js/reader.js b/src/pkg/dashboard/static-src/js/reader.js index 5a3130d774..618160da1d 100644 --- a/src/pkg/dashboard/static-src/js/reader.js +++ b/src/pkg/dashboard/static-src/js/reader.js @@ -13,9 +13,8 @@ if (!window.HIVE_READER || document.getElementById('hive-reader')) return; // Cards kept on screen, regions waiting to be read, and how long one // paragraph stays in focus before the next arrives. - var MAX_CARDS = 60, MAX_PENDING = 40, BASE_DWELL = 5000, LINE_DWELL = 600, MAX_DWELL = 30000; + var MAX_CARDS = 32, MAX_LINES = 1200, MAX_BYTES = 256 * 1024, BASE_DWELL = 5000, LINE_DWELL = 600, MAX_DWELL = 30000; var NAV_KEYS = ['ArrowUp', 'ArrowDown', 'PageUp', 'PageDown', 'Home', 'End']; - var STAGES = {'pushed-lines': 'pushed'}; var root = document.createElement('main'); root.id = 'hive-reader'; root.setAttribute('aria-label', 'Code activity reader'); @@ -35,32 +34,44 @@ button.setAttribute('type', 'button'); button.setAttribute('aria-pressed', 'false'); button.setAttribute('title', 'Pause or resume (P / Space)'); + var replay = element('button', 'reader-pause reader-replay', 'Replay buffered snapshot', bar); + replay.setAttribute('type', 'button'); + replay.setAttribute('title', 'Explicit replay of retained changes; off by default'); + var freshness = element('span', 'reader-freshness', 'Source has not completed an ingest pass', bar); var scroller = element('div', 'reader-scroll'); scroller.setAttribute('tabindex', '0'); var feed = element('div', 'reader-feed', '', scroller); var motion = window.matchMedia('(prefers-reduced-motion: reduce)'); var pending = [], current = null, readUntil = 0, timer = 0, lastSeq = 0; + var history = [], retainedLines = 0, retainedBytes = 0, epoch = '', snapshotThrough = null, contentGeneration = 0; var enabled = false, manualPause = false, loading = false, again = false, problem = ''; function paused(){ return manualPause || document.hidden; } - function regionKey(line){ return [line.repo, line.pr, line.head || '', line.path].join('\u0000'); } - // Consecutive lines of one file at one head form one contiguous region; - // a gap in seq (a dropped line or another file) starts a new one. Only - // unread regions grow, so a paragraph on screen never changes. - function group(lines, stage){ - lines.forEach(function(line){ - var last = pending[pending.length - 1], seq = Number(line.seq) || 0; - if (last && last.key === regionKey(line) && seq && seq === last.lastSeq + 1) { - last.lines.push(line); last.lastSeq = seq; last.at = line.at; - return; + // The source supplies indivisible regions. Never infer a paragraph from + // neighbouring sequence numbers or merge across context/page boundaries. + function group(regions){ + regions.forEach(function(region){ + if (!region || !Array.isArray(region.lines) || !region.lines.length) return; + if (!region.lines.every(function(line){ return line && typeof line.line === 'string' && (line.op === '+' || line.op === '-'); })) return; + var size = region.lines.reduce(function(n, line){ return n + line.line.length * 2; }, 0); + if (region.lines.length > MAX_LINES || size > MAX_BYTES) return; + var item = {repo: region.repo, pr: region.pr, path: region.path, prURL: region.pr_url, + stage: 'Pushed PR', at: region.at, lines: region.lines, size: size, + delivery: Number(region.seq) <= snapshotThrough ? 'Buffered snapshot' : 'Newly received checkpoint'}; + history.push(item); pending.push(item); + retainedLines += item.lines.length; retainedBytes += size; + while (retainedLines > MAX_LINES || retainedBytes > MAX_BYTES) { + var victim = history.findIndex(function(r){ return r !== current; }); + var old = history.splice(victim, 1)[0]; + retainedLines -= old.lines.length; retainedBytes -= old.size; + pending = pending.filter(function(r){ return r !== old; }); + if (old.node) { feed.removeChild(old.node); old.node = null; } + if (current === old) current = null; } - pending.push({key: regionKey(line), repo: line.repo, pr: Number(line.pr) || 0, path: line.path, - stage: stage, prUrl: line.pr_url, lines: [line], lastSeq: seq, at: line.at}); }); - if (pending.length > MAX_PENDING) pending = pending.slice(-MAX_PENDING); } - // Prefer an https pr_url from the source; otherwise derive the link from repo and PR number. function prLink(region){ - if (typeof region.prUrl === 'string' && /^https:\/\/[^\s<>"']+$/.test(region.prUrl)) return region.prUrl; + var url = typeof region.prURL === 'string' ? region.prURL : ''; + if (/^https:\/\/[^\s"'<>]+\/pull\/\d+$/.test(url)) return url; if (/^[\w.-]+\/[\w.-]+$/.test(region.repo || '') && region.pr > 0) { return 'https://github.com/' + region.repo + '/pull/' + region.pr; } @@ -87,14 +98,15 @@ } element('span', 'reader-file', region.path, meta); element('span', 'reader-stage', region.stage, meta); + element('span', 'reader-delivery', region.delivery, meta); var when = observed(region.at); var time = element('time', 'reader-time', when.text, meta); if (when.iso) time.setAttribute('datetime', when.iso); var diff = element('pre', 'reader-diff', '', article); region.lines.forEach(function(line){ - var del = line.op === '-'; - var row = element('span', 'reader-line ' + (del ? 'reader-del' : 'reader-add'), '', diff); - row.textContent = (del ? '\u2212' : line.op || '+') + ' ' + line.line + '\n'; + var removed = line.op === '-'; + var row = element('span', 'reader-line ' + (removed ? 'reader-del' : 'reader-add'), '', diff); + row.textContent = (removed ? '\u2212 ' : '+ ') + line.line + '\n'; }); return article; } @@ -104,6 +116,7 @@ while (feed.children.length > MAX_CARDS) { var first = feed.children[0], height = first.offsetHeight || 0; feed.removeChild(first); + history.forEach(function(r){ if (r.node === first) r.node = null; }); scroller.scrollTop = Math.max(0, (scroller.scrollTop || 0) - height); } } @@ -119,7 +132,7 @@ var text = problem; if (!text && enabled) { if (pending.length) text = pending.length + (pending.length === 1 ? ' change' : ' changes') + ' waiting'; - else text = current ? 'Caught up · holding until new changes are pushed' : 'Waiting for pushed changes…'; + else text = current ? 'Caught up with available checkpoints · holding' : 'No buffered checkpoints · source may still be warming or filtering'; } if (paused()) text = 'paused' + (text ? ' · ' + text : ''); status.textContent = text; @@ -165,51 +178,55 @@ } function inspect(){ setPaused(true); } function clearContent(message){ + contentGeneration++; stop(); - enabled = false; pending = []; current = null; lastSeq = 0; problem = message; + enabled = false; pending = []; history = []; retainedLines = 0; retainedBytes = 0; + current = null; lastSeq = 0; snapshotThrough = null; problem = message; feed.replaceChildren(); updateStatus(); } pull = function(){ + if (document.hidden) return; if (loading) { again = true; return; } loading = true; again = false; - fetch('/api/matrix/lines?since=' + lastSeq, {cache: 'no-store', credentials: 'same-origin'}) + var requestGeneration = contentGeneration; + fetch('/api/matrix/lines?since=' + lastSeq + '&format=regions', {cache: 'no-store', credentials: 'same-origin'}) .then(function(response){ + if (requestGeneration !== contentGeneration || document.hidden) return null; if (response.status === 404) { clearContent('The code activity reader is not enabled on this hive'); return null; } if (response.status === 401 || response.status === 403) { clearContent('Sign in with permission to read code activity'); return null; } if (!response.ok) throw new Error('reader pull failed'); return response.json(); }).then(function(data){ - if (!data) return; + if (!data || requestGeneration !== contentGeneration || document.hidden) return; var seq = Number(data.seq) || 0; - // A restarted hive can reset its sequence; start over cleanly. - if (seq < lastSeq) { stop(); pending = []; current = null; lastSeq = 0; feed.replaceChildren(); } - var raw = Array.isArray(data.lines) ? data.lines : []; - var incoming = raw.filter(function(line){ - return line && typeof line.line === 'string' && typeof line.repo === 'string' && typeof line.path === 'string'; - }); - var source = typeof data.source === 'string' ? data.source : ''; - group(incoming, STAGES[source] || source || 'pushed'); - // data.seq is the newest line the hive holds, not the last one in - // this page: the server caps a page, so resume from the last line - // returned and pull again until the page reaches the newest seq. - var returned = raw.length ? Number(raw[raw.length - 1] && raw[raw.length - 1].seq) || 0 : 0; - var more = returned > 0 && returned < seq; - lastSeq = more ? returned : seq; + if (data.epoch !== epoch || seq < lastSeq) { + clearContent(''); epoch = data.epoch; + } + if (snapshotThrough === null) snapshotThrough = Number(data.latest) || 0; + group(Array.isArray(data.regions) ? data.regions : []); + freshness.textContent = data.last_ingest_at + ? 'Last source pass: ' + data.last_ingest_at + ' · next expected in ' + (data.next_ingest_in_s || 0) + 's' + : 'Source has not completed an ingest pass · empty does not mean no work'; + if (seq < Number(data.latest)) again = true; + lastSeq = seq; enabled = true; problem = ''; - // Hold the reveal while catching up so a region split across - // pages still arrives as one paragraph. - if (more) { again = true; updateStatus(); return; } schedule(false); - }).catch(function(){ problem = 'Code activity unavailable · retrying'; updateStatus(); }) + }).catch(function(){ clearContent('Code activity unavailable · retrying'); }) .finally(function(){ loading = false; if (again) pull(); }); }; + replay.addEventListener('click', function(){ + if (!enabled || !history.length) return; + stop(); feed.replaceChildren(); current = null; readUntil = 0; + history.forEach(function(r){ r.node = null; r.delivery = 'Replay · buffered snapshot'; }); + pending = history.slice(); schedule(false); + }); button.addEventListener('click', function(){ setPaused(!manualPause); }); scroller.addEventListener('wheel', inspect, {passive: true}); scroller.addEventListener('touchstart', inspect, {passive: true}); document.addEventListener('visibilitychange', function(){ - if (document.hidden) { stop(); updateStatus(); return; } - schedule(true); pull(); + if (document.hidden) { clearContent('Hidden · buffered code cleared'); return; } + pull(); }); document.addEventListener('keydown', function(event){ var target = event.target || {}; diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 3d9a4bdb95..7144da7c60 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -7503,9 +7503,8 @@ if (!window.HIVE_READER || document.getElementById('hive-reader')) return; // Cards kept on screen, regions waiting to be read, and how long one // paragraph stays in focus before the next arrives. - var MAX_CARDS = 60, MAX_PENDING = 40, BASE_DWELL = 5000, LINE_DWELL = 600, MAX_DWELL = 30000; + var MAX_CARDS = 32, MAX_LINES = 1200, MAX_BYTES = 256 * 1024, BASE_DWELL = 5000, LINE_DWELL = 600, MAX_DWELL = 30000; var NAV_KEYS = ['ArrowUp', 'ArrowDown', 'PageUp', 'PageDown', 'Home', 'End']; - var STAGES = {'pushed-lines': 'pushed'}; var root = document.createElement('main'); root.id = 'hive-reader'; root.setAttribute('aria-label', 'Code activity reader'); @@ -7525,32 +7524,44 @@ button.setAttribute('type', 'button'); button.setAttribute('aria-pressed', 'false'); button.setAttribute('title', 'Pause or resume (P / Space)'); + var replay = element('button', 'reader-pause reader-replay', 'Replay buffered snapshot', bar); + replay.setAttribute('type', 'button'); + replay.setAttribute('title', 'Explicit replay of retained changes; off by default'); + var freshness = element('span', 'reader-freshness', 'Source has not completed an ingest pass', bar); var scroller = element('div', 'reader-scroll'); scroller.setAttribute('tabindex', '0'); var feed = element('div', 'reader-feed', '', scroller); var motion = window.matchMedia('(prefers-reduced-motion: reduce)'); var pending = [], current = null, readUntil = 0, timer = 0, lastSeq = 0; + var history = [], retainedLines = 0, retainedBytes = 0, epoch = '', snapshotThrough = null, contentGeneration = 0; var enabled = false, manualPause = false, loading = false, again = false, problem = ''; function paused(){ return manualPause || document.hidden; } - function regionKey(line){ return [line.repo, line.pr, line.head || '', line.path].join('\u0000'); } - // Consecutive lines of one file at one head form one contiguous region; - // a gap in seq (a dropped line or another file) starts a new one. Only - // unread regions grow, so a paragraph on screen never changes. - function group(lines, stage){ - lines.forEach(function(line){ - var last = pending[pending.length - 1], seq = Number(line.seq) || 0; - if (last && last.key === regionKey(line) && seq && seq === last.lastSeq + 1) { - last.lines.push(line); last.lastSeq = seq; last.at = line.at; - return; + // The source supplies indivisible regions. Never infer a paragraph from + // neighbouring sequence numbers or merge across context/page boundaries. + function group(regions){ + regions.forEach(function(region){ + if (!region || !Array.isArray(region.lines) || !region.lines.length) return; + if (!region.lines.every(function(line){ return line && typeof line.line === 'string' && (line.op === '+' || line.op === '-'); })) return; + var size = region.lines.reduce(function(n, line){ return n + line.line.length * 2; }, 0); + if (region.lines.length > MAX_LINES || size > MAX_BYTES) return; + var item = {repo: region.repo, pr: region.pr, path: region.path, prURL: region.pr_url, + stage: 'Pushed PR', at: region.at, lines: region.lines, size: size, + delivery: Number(region.seq) <= snapshotThrough ? 'Buffered snapshot' : 'Newly received checkpoint'}; + history.push(item); pending.push(item); + retainedLines += item.lines.length; retainedBytes += size; + while (retainedLines > MAX_LINES || retainedBytes > MAX_BYTES) { + var victim = history.findIndex(function(r){ return r !== current; }); + var old = history.splice(victim, 1)[0]; + retainedLines -= old.lines.length; retainedBytes -= old.size; + pending = pending.filter(function(r){ return r !== old; }); + if (old.node) { feed.removeChild(old.node); old.node = null; } + if (current === old) current = null; } - pending.push({key: regionKey(line), repo: line.repo, pr: Number(line.pr) || 0, path: line.path, - stage: stage, prUrl: line.pr_url, lines: [line], lastSeq: seq, at: line.at}); }); - if (pending.length > MAX_PENDING) pending = pending.slice(-MAX_PENDING); } - // Prefer an https pr_url from the source; otherwise derive the link from repo and PR number. function prLink(region){ - if (typeof region.prUrl === 'string' && /^https:\/\/[^\s<>"']+$/.test(region.prUrl)) return region.prUrl; + var url = typeof region.prURL === 'string' ? region.prURL : ''; + if (/^https:\/\/[^\s"'<>]+\/pull\/\d+$/.test(url)) return url; if (/^[\w.-]+\/[\w.-]+$/.test(region.repo || '') && region.pr > 0) { return 'https://github.com/' + region.repo + '/pull/' + region.pr; } @@ -7577,14 +7588,15 @@ } element('span', 'reader-file', region.path, meta); element('span', 'reader-stage', region.stage, meta); + element('span', 'reader-delivery', region.delivery, meta); var when = observed(region.at); var time = element('time', 'reader-time', when.text, meta); if (when.iso) time.setAttribute('datetime', when.iso); var diff = element('pre', 'reader-diff', '', article); region.lines.forEach(function(line){ - var del = line.op === '-'; - var row = element('span', 'reader-line ' + (del ? 'reader-del' : 'reader-add'), '', diff); - row.textContent = (del ? '\u2212' : line.op || '+') + ' ' + line.line + '\n'; + var removed = line.op === '-'; + var row = element('span', 'reader-line ' + (removed ? 'reader-del' : 'reader-add'), '', diff); + row.textContent = (removed ? '\u2212 ' : '+ ') + line.line + '\n'; }); return article; } @@ -7594,6 +7606,7 @@ while (feed.children.length > MAX_CARDS) { var first = feed.children[0], height = first.offsetHeight || 0; feed.removeChild(first); + history.forEach(function(r){ if (r.node === first) r.node = null; }); scroller.scrollTop = Math.max(0, (scroller.scrollTop || 0) - height); } } @@ -7609,7 +7622,7 @@ var text = problem; if (!text && enabled) { if (pending.length) text = pending.length + (pending.length === 1 ? ' change' : ' changes') + ' waiting'; - else text = current ? 'Caught up · holding until new changes are pushed' : 'Waiting for pushed changes…'; + else text = current ? 'Caught up with available checkpoints · holding' : 'No buffered checkpoints · source may still be warming or filtering'; } if (paused()) text = 'paused' + (text ? ' · ' + text : ''); status.textContent = text; @@ -7655,51 +7668,55 @@ } function inspect(){ setPaused(true); } function clearContent(message){ + contentGeneration++; stop(); - enabled = false; pending = []; current = null; lastSeq = 0; problem = message; + enabled = false; pending = []; history = []; retainedLines = 0; retainedBytes = 0; + current = null; lastSeq = 0; snapshotThrough = null; problem = message; feed.replaceChildren(); updateStatus(); } pull = function(){ + if (document.hidden) return; if (loading) { again = true; return; } loading = true; again = false; - fetch('/api/matrix/lines?since=' + lastSeq, {cache: 'no-store', credentials: 'same-origin'}) + var requestGeneration = contentGeneration; + fetch('/api/matrix/lines?since=' + lastSeq + '&format=regions', {cache: 'no-store', credentials: 'same-origin'}) .then(function(response){ + if (requestGeneration !== contentGeneration || document.hidden) return null; if (response.status === 404) { clearContent('The code activity reader is not enabled on this hive'); return null; } if (response.status === 401 || response.status === 403) { clearContent('Sign in with permission to read code activity'); return null; } if (!response.ok) throw new Error('reader pull failed'); return response.json(); }).then(function(data){ - if (!data) return; + if (!data || requestGeneration !== contentGeneration || document.hidden) return; var seq = Number(data.seq) || 0; - // A restarted hive can reset its sequence; start over cleanly. - if (seq < lastSeq) { stop(); pending = []; current = null; lastSeq = 0; feed.replaceChildren(); } - var raw = Array.isArray(data.lines) ? data.lines : []; - var incoming = raw.filter(function(line){ - return line && typeof line.line === 'string' && typeof line.repo === 'string' && typeof line.path === 'string'; - }); - var source = typeof data.source === 'string' ? data.source : ''; - group(incoming, STAGES[source] || source || 'pushed'); - // data.seq is the newest line the hive holds, not the last one in - // this page: the server caps a page, so resume from the last line - // returned and pull again until the page reaches the newest seq. - var returned = raw.length ? Number(raw[raw.length - 1] && raw[raw.length - 1].seq) || 0 : 0; - var more = returned > 0 && returned < seq; - lastSeq = more ? returned : seq; + if (data.epoch !== epoch || seq < lastSeq) { + clearContent(''); epoch = data.epoch; + } + if (snapshotThrough === null) snapshotThrough = Number(data.latest) || 0; + group(Array.isArray(data.regions) ? data.regions : []); + freshness.textContent = data.last_ingest_at + ? 'Last source pass: ' + data.last_ingest_at + ' · next expected in ' + (data.next_ingest_in_s || 0) + 's' + : 'Source has not completed an ingest pass · empty does not mean no work'; + if (seq < Number(data.latest)) again = true; + lastSeq = seq; enabled = true; problem = ''; - // Hold the reveal while catching up so a region split across - // pages still arrives as one paragraph. - if (more) { again = true; updateStatus(); return; } schedule(false); - }).catch(function(){ problem = 'Code activity unavailable · retrying'; updateStatus(); }) + }).catch(function(){ clearContent('Code activity unavailable · retrying'); }) .finally(function(){ loading = false; if (again) pull(); }); }; + replay.addEventListener('click', function(){ + if (!enabled || !history.length) return; + stop(); feed.replaceChildren(); current = null; readUntil = 0; + history.forEach(function(r){ r.node = null; r.delivery = 'Replay · buffered snapshot'; }); + pending = history.slice(); schedule(false); + }); button.addEventListener('click', function(){ setPaused(!manualPause); }); scroller.addEventListener('wheel', inspect, {passive: true}); scroller.addEventListener('touchstart', inspect, {passive: true}); document.addEventListener('visibilitychange', function(){ - if (document.hidden) { stop(); updateStatus(); return; } - schedule(true); pull(); + if (document.hidden) { clearContent('Hidden · buffered code cleared'); return; } + pull(); }); document.addEventListener('keydown', function(event){ var target = event.target || {}; From c3a529d425185f3222c11b90bcefe1ad3dbe9dda Mon Sep 17 00:00:00 2001 From: thatguywob Date: Wed, 4 Nov 2026 01:27:26 +0000 Subject: [PATCH 2/6] =?UTF-8?q?=F0=9F=93=96=20docs:=20refresh=20dashboard?= =?UTF-8?q?=20route=20source=20citations?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: thatguywob --- src/docs/api-reference/auth.md | 6 ++--- src/docs/api-reference/dashboard-misc.md | 2 +- .../api-reference/health-status-events.md | 22 +++++++++---------- src/docs/api-reference/snapshots-and-style.md | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 78ee568d5b..f53613dd29 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1663` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1664` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1665` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1662` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1663` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1664` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 317eccf99b..74debd9a15 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -80,7 +80,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:377` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:378` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:379` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1670` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1669` | ### Activity observation provenance The CI snapshot retains real check/node IDs, head, App ID and transition times. diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index 5458c2d3fb..5e9728defb 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:44` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:45` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:69` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1643` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1644` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1645` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1651` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1656` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1657` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1658` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1659` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1660` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1661` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1662` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1642` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1643` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1644` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1650` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot | `pkg/dashboard/server.go:1655` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1656` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1657` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1658` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1659` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1660` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1661` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 6139d7dc10..2a7f4e28bc 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:46` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1733` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1732` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:82` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:83` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:84` | From 4d8855c3f01ef61a0e6a7ae7a61aad51e92a80fe Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 22:02:27 -0500 Subject: [PATCH 3/6] =?UTF-8?q?=F0=9F=90=9B=20fix(dashboard):=20settle=20c?= =?UTF-8?q?heckpoint=20nudge=20before=20policy=20edits=20in=20tests?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI on this branch was red on four fronts: - TestCheckpointAPIAuthColdAndEpoch raced: the observer goroutine reads s.deps.Config.Dashboard.Matrix while broadcasting the code-lines nudge, and the test rewrote that config as soon as the region was ingested. Register an SSE client and wait for the nudge frame instead, so the observer is quiescent before the policy edits. - TestSleepRatchet: both new polling loops used time.Sleep; the above removes one and the persist test now uses the existing waitFor helper. - TestDashboardHistoryPushSitesAreCapped: the reader's retained-region array matched the history-array guard. It is bounded by line/byte budget rather than capHistory, so name it `retained`. - mobile-shots tracker-header-geometry: inherited from the prerequisite branch; merged scanner/fix-16475 which carries the compact mobile chevron fix (7f93822f5). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/pkg/dashboard/checkpoint_source_test.go | 26 +++++++++++++++------ src/pkg/dashboard/matrix_persist_test.go | 11 +-------- src/pkg/dashboard/static-src/js/reader.js | 18 +++++++------- src/pkg/dashboard/static/index.html | 18 +++++++------- 4 files changed, 38 insertions(+), 35 deletions(-) diff --git a/src/pkg/dashboard/checkpoint_source_test.go b/src/pkg/dashboard/checkpoint_source_test.go index 58b4df0908..3474434e53 100644 --- a/src/pkg/dashboard/checkpoint_source_test.go +++ b/src/pkg/dashboard/checkpoint_source_test.go @@ -240,15 +240,27 @@ func TestCheckpointAPIAuthColdAndEpoch(t *testing.T) { t.Fatalf("cold response %d %s", rr.Code, rr.Body.String()) } // Feed the real observer queue, including a deletion-only checkpoint. + // Wait for the nudge frame rather than the ingest: the observer goroutine + // reads the matrix config while broadcasting, so the policy edits below + // must not start until that frame has been delivered. + frames := make(chan []byte, 8) + s.sseMu.Lock() + s.sseClients[frames] = struct{}{} + s.sseMu.Unlock() s.MatrixObserver(cfg)(matrixObs(1, "real-head", sentinel.File{Path: "a.go", Patch: "@@ -1,2 +0,0 @@\n-old\n-old\n"})) - deadline := time.After(2 * time.Second) - for len(s.matrix.checkpoints.since(0, 1).Regions) == 0 { - select { - case <-deadline: - t.Fatal("observer did not feed checkpoint source") - default: - time.Sleep(time.Millisecond) + select { + case frame := <-frames: + if string(frame) != matrixCodeLinesSSEFrame(0) { + t.Fatalf("unexpected frame %q", frame) } + case <-time.After(2 * time.Second): + t.Fatal("observer did not feed checkpoint source") + } + s.sseMu.Lock() + delete(s.sseClients, frames) + s.sseMu.Unlock() + if len(s.matrix.checkpoints.since(0, 1).Regions) == 0 { + t.Fatal("checkpoint source empty after nudge") } s.RecordMatrixIngest(time.Now().UTC(), time.Minute) rr = request("Bearer checkpoint-test-token", "") diff --git a/src/pkg/dashboard/matrix_persist_test.go b/src/pkg/dashboard/matrix_persist_test.go index 9489fd28b7..58aeb72b8d 100644 --- a/src/pkg/dashboard/matrix_persist_test.go +++ b/src/pkg/dashboard/matrix_persist_test.go @@ -4,7 +4,6 @@ import ( "os" "path/filepath" "testing" - "time" "github.com/hivecommons/hive/pkg/sentinel" ) @@ -26,15 +25,7 @@ func TestMatrixCodeHistoryIsMemoryOnly(t *testing.T) { t.Fatal("legacy persistent code restored") } observe(matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: checkpointPatch("fresh checkpoint")})) - deadline := time.After(2 * time.Second) - for s.matrix.Latest() == 0 { - select { - case <-deadline: - t.Fatal("observer did not ingest") - default: - time.Sleep(time.Millisecond) - } - } + waitFor(t, func() bool { return s.matrix.Latest() != 0 }, "observer ingest") s.SetMatrixPersistPath(path) got, err := os.ReadFile(path) if err != nil || string(got) != string(seed) { diff --git a/src/pkg/dashboard/static-src/js/reader.js b/src/pkg/dashboard/static-src/js/reader.js index 618160da1d..58a1d473fb 100644 --- a/src/pkg/dashboard/static-src/js/reader.js +++ b/src/pkg/dashboard/static-src/js/reader.js @@ -43,7 +43,7 @@ var feed = element('div', 'reader-feed', '', scroller); var motion = window.matchMedia('(prefers-reduced-motion: reduce)'); var pending = [], current = null, readUntil = 0, timer = 0, lastSeq = 0; - var history = [], retainedLines = 0, retainedBytes = 0, epoch = '', snapshotThrough = null, contentGeneration = 0; + var retained = [], retainedLines = 0, retainedBytes = 0, epoch = '', snapshotThrough = null, contentGeneration = 0; var enabled = false, manualPause = false, loading = false, again = false, problem = ''; function paused(){ return manualPause || document.hidden; } // The source supplies indivisible regions. Never infer a paragraph from @@ -57,11 +57,11 @@ var item = {repo: region.repo, pr: region.pr, path: region.path, prURL: region.pr_url, stage: 'Pushed PR', at: region.at, lines: region.lines, size: size, delivery: Number(region.seq) <= snapshotThrough ? 'Buffered snapshot' : 'Newly received checkpoint'}; - history.push(item); pending.push(item); + retained.push(item); pending.push(item); retainedLines += item.lines.length; retainedBytes += size; while (retainedLines > MAX_LINES || retainedBytes > MAX_BYTES) { - var victim = history.findIndex(function(r){ return r !== current; }); - var old = history.splice(victim, 1)[0]; + var victim = retained.findIndex(function(r){ return r !== current; }); + var old = retained.splice(victim, 1)[0]; retainedLines -= old.lines.length; retainedBytes -= old.size; pending = pending.filter(function(r){ return r !== old; }); if (old.node) { feed.removeChild(old.node); old.node = null; } @@ -116,7 +116,7 @@ while (feed.children.length > MAX_CARDS) { var first = feed.children[0], height = first.offsetHeight || 0; feed.removeChild(first); - history.forEach(function(r){ if (r.node === first) r.node = null; }); + retained.forEach(function(r){ if (r.node === first) r.node = null; }); scroller.scrollTop = Math.max(0, (scroller.scrollTop || 0) - height); } } @@ -180,7 +180,7 @@ function clearContent(message){ contentGeneration++; stop(); - enabled = false; pending = []; history = []; retainedLines = 0; retainedBytes = 0; + enabled = false; pending = []; retained = []; retainedLines = 0; retainedBytes = 0; current = null; lastSeq = 0; snapshotThrough = null; problem = message; feed.replaceChildren(); updateStatus(); @@ -216,10 +216,10 @@ .finally(function(){ loading = false; if (again) pull(); }); }; replay.addEventListener('click', function(){ - if (!enabled || !history.length) return; + if (!enabled || !retained.length) return; stop(); feed.replaceChildren(); current = null; readUntil = 0; - history.forEach(function(r){ r.node = null; r.delivery = 'Replay · buffered snapshot'; }); - pending = history.slice(); schedule(false); + retained.forEach(function(r){ r.node = null; r.delivery = 'Replay · buffered snapshot'; }); + pending = retained.slice(); schedule(false); }); button.addEventListener('click', function(){ setPaused(!manualPause); }); scroller.addEventListener('wheel', inspect, {passive: true}); diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 7144da7c60..d32c59f71a 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -7533,7 +7533,7 @@ var feed = element('div', 'reader-feed', '', scroller); var motion = window.matchMedia('(prefers-reduced-motion: reduce)'); var pending = [], current = null, readUntil = 0, timer = 0, lastSeq = 0; - var history = [], retainedLines = 0, retainedBytes = 0, epoch = '', snapshotThrough = null, contentGeneration = 0; + var retained = [], retainedLines = 0, retainedBytes = 0, epoch = '', snapshotThrough = null, contentGeneration = 0; var enabled = false, manualPause = false, loading = false, again = false, problem = ''; function paused(){ return manualPause || document.hidden; } // The source supplies indivisible regions. Never infer a paragraph from @@ -7547,11 +7547,11 @@ var item = {repo: region.repo, pr: region.pr, path: region.path, prURL: region.pr_url, stage: 'Pushed PR', at: region.at, lines: region.lines, size: size, delivery: Number(region.seq) <= snapshotThrough ? 'Buffered snapshot' : 'Newly received checkpoint'}; - history.push(item); pending.push(item); + retained.push(item); pending.push(item); retainedLines += item.lines.length; retainedBytes += size; while (retainedLines > MAX_LINES || retainedBytes > MAX_BYTES) { - var victim = history.findIndex(function(r){ return r !== current; }); - var old = history.splice(victim, 1)[0]; + var victim = retained.findIndex(function(r){ return r !== current; }); + var old = retained.splice(victim, 1)[0]; retainedLines -= old.lines.length; retainedBytes -= old.size; pending = pending.filter(function(r){ return r !== old; }); if (old.node) { feed.removeChild(old.node); old.node = null; } @@ -7606,7 +7606,7 @@ while (feed.children.length > MAX_CARDS) { var first = feed.children[0], height = first.offsetHeight || 0; feed.removeChild(first); - history.forEach(function(r){ if (r.node === first) r.node = null; }); + retained.forEach(function(r){ if (r.node === first) r.node = null; }); scroller.scrollTop = Math.max(0, (scroller.scrollTop || 0) - height); } } @@ -7670,7 +7670,7 @@ function clearContent(message){ contentGeneration++; stop(); - enabled = false; pending = []; history = []; retainedLines = 0; retainedBytes = 0; + enabled = false; pending = []; retained = []; retainedLines = 0; retainedBytes = 0; current = null; lastSeq = 0; snapshotThrough = null; problem = message; feed.replaceChildren(); updateStatus(); @@ -7706,10 +7706,10 @@ .finally(function(){ loading = false; if (again) pull(); }); }; replay.addEventListener('click', function(){ - if (!enabled || !history.length) return; + if (!enabled || !retained.length) return; stop(); feed.replaceChildren(); current = null; readUntil = 0; - history.forEach(function(r){ r.node = null; r.delivery = 'Replay · buffered snapshot'; }); - pending = history.slice(); schedule(false); + retained.forEach(function(r){ r.node = null; r.delivery = 'Replay · buffered snapshot'; }); + pending = retained.slice(); schedule(false); }); button.addEventListener('click', function(){ setPaused(!manualPause); }); scroller.addEventListener('wheel', inspect, {passive: true}); From 961e76e7c8d236bf1904e5cb2da38827c180becd Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:30:04 -0500 Subject: [PATCH 4/6] =?UTF-8?q?=F0=9F=90=9B=20fix(dashboard):=20let=20conc?= =?UTF-8?q?urrent=20matrix=20observers=20share=20the=20policy=20lock?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Observe used policyMu.TryLock, so two sentinel observers running at once (the sweep and webhook paths each run observers in their own goroutine) made the loser count as dropped and never re-observed. policyMu is now an RWMutex: observers take TryRLock and only back off while setConfig holds the exclusive lock, keeping Observe non-blocking. Hive-Fix-Attempt: 2/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/pkg/dashboard/checkpoint_source_test.go | 23 +++++++++++++++++++++ src/pkg/dashboard/matrix_source.go | 15 +++++++++----- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/pkg/dashboard/checkpoint_source_test.go b/src/pkg/dashboard/checkpoint_source_test.go index 3474434e53..8cb36495dd 100644 --- a/src/pkg/dashboard/checkpoint_source_test.go +++ b/src/pkg/dashboard/checkpoint_source_test.go @@ -166,6 +166,29 @@ func TestCheckpointPolicyClearsSharedQueue(t *testing.T) { } +func TestCheckpointConcurrentObserversNotDropped(t *testing.T) { + m := newMatrixSource(matrixTestConfig(), nil) + // Another observer in flight holds the shared side of the policy lock. + m.policyMu.RLock() + defer m.policyMu.RUnlock() + done := make(chan struct{}) + go func() { + m.Observe(matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: checkpointPatch("concurrent code")})) + close(done) + }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("observer blocked on a concurrent observer") + } + if got := m.Dropped(); got != 0 { + t.Fatalf("Dropped() = %d, want 0: concurrent observer discarded", got) + } + if len(m.in) != 1 { + t.Fatal("concurrent observation did not enter queue") + } +} + func TestCheckpointWholeBounds(t *testing.T) { c := newCheckpointSource(matrixTestConfig()) now := time.Now() diff --git a/src/pkg/dashboard/matrix_source.go b/src/pkg/dashboard/matrix_source.go index 92b2c709cb..4e96d59e75 100644 --- a/src/pkg/dashboard/matrix_source.go +++ b/src/pkg/dashboard/matrix_source.go @@ -62,7 +62,10 @@ type matrixQueuedObservation struct { // matrixSource owns the shared observer queue and its legacy line projection. type matrixSource struct { checkpoints *checkpointSource - policyMu sync.Mutex + // policyMu is held exclusively by setConfig and shared by observers, so + // concurrent Observe calls never contend with each other; only a policy + // update in flight makes an observer back off. + policyMu sync.RWMutex queuedEpoch atomic.Pointer[string] cfg atomic.Pointer[config.MatrixConfig] in chan matrixQueuedObservation @@ -146,14 +149,16 @@ func (m *matrixSource) admits(obs github.SentinelPatchObservation) bool { } // Observe is the sentinel PatchObserver. It gates the observation, then hands -// it off without blocking: when the ingest channel is full the observation -// is dropped and counted. +// it off without blocking: when a policy update is in flight or the ingest +// channel is full the observation is dropped and counted. Observers hold the +// policy read lock, so sweeps running many observers at once do not drop +// each other. func (m *matrixSource) Observe(obs github.SentinelPatchObservation) { - if !m.policyMu.TryLock() { + if !m.policyMu.TryRLock() { m.dropped.Add(1) return } - defer m.policyMu.Unlock() + defer m.policyMu.RUnlock() if !m.admits(obs) { return } From 12549ec5d96d55beacbb7485bb77b7f52d65605b Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 01:55:18 -0500 Subject: [PATCH 5/6] =?UTF-8?q?=F0=9F=90=9B=20fix(dashboard):=20skip=20the?= =?UTF-8?q?=20matrix=20policy=20write-lock=20when=20the=20policy=20is=20un?= =?UTF-8?q?changed?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit handleCheckpointRegions refreshes the Matrix policy on every reader GET, and setConfig took policyMu exclusively even when nothing changed. An observer arriving during that window failed TryRLock and was dropped, and because the checkpoint head map is keyed by head SHA the dropped head was never re-observed. setConfig now compares the incoming policy with the one in force and returns before locking when they match. The stale-epoch skip in run() now documents why the legacy projection also drops the observation: setConfig reset that backlog for the same policy change. Hive-Fix-Attempt: 3/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- src/pkg/dashboard/checkpoint_source_test.go | 26 +++++++++++++++++++++ src/pkg/dashboard/matrix_source.go | 21 +++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/src/pkg/dashboard/checkpoint_source_test.go b/src/pkg/dashboard/checkpoint_source_test.go index 8cb36495dd..c149c56c14 100644 --- a/src/pkg/dashboard/checkpoint_source_test.go +++ b/src/pkg/dashboard/checkpoint_source_test.go @@ -189,6 +189,32 @@ func TestCheckpointConcurrentObserversNotDropped(t *testing.T) { } } +func TestCheckpointUnchangedPolicyDoesNotBlockObservers(t *testing.T) { + m := newMatrixSource(matrixTestConfig(), nil) + epoch := m.checkpoints.policyEpoch() + // An observer in flight holds the shared side of the policy lock; the + // reader poll re-applying the same policy must not wait for or evict it. + m.policyMu.RLock() + defer m.policyMu.RUnlock() + done := make(chan struct{}) + go func() { m.setConfig(matrixTestConfig()); close(done) }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("unchanged policy refresh took the exclusive policy lock") + } + if m.checkpoints.policyEpoch() != epoch { + t.Fatal("unchanged policy refresh rotated the epoch") + } + m.Observe(matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: checkpointPatch("polled code")})) + if got := m.Dropped(); got != 0 { + t.Fatalf("Dropped() = %d, want 0: observer dropped during an unchanged policy refresh", got) + } + if len(m.in) != 1 { + t.Fatal("observation did not enter queue") + } +} + func TestCheckpointWholeBounds(t *testing.T) { c := newCheckpointSource(matrixTestConfig()) now := time.Now() diff --git a/src/pkg/dashboard/matrix_source.go b/src/pkg/dashboard/matrix_source.go index 4e96d59e75..1d23c6c005 100644 --- a/src/pkg/dashboard/matrix_source.go +++ b/src/pkg/dashboard/matrix_source.go @@ -3,6 +3,7 @@ package dashboard import ( "crypto/sha256" "log/slog" + "slices" "strconv" "sync" "sync/atomic" @@ -115,9 +116,24 @@ func newMatrixSource(cfg config.MatrixConfig, logger *slog.Logger) *matrixSource return m } +// samePolicy reports whether cfg matches the policy already in force. The +// reader poll refreshes the policy on every GET, so an unchanged policy must +// not take the exclusive lock that makes observers back off. +func (m *matrixSource) samePolicy(cfg config.MatrixConfig) bool { + cur := m.cfg.Load() + return cur != nil && m.queuedEpoch.Load() != nil && cur.Enabled == cfg.Enabled && + slices.Equal(cur.Repos, cfg.Repos) && slices.Equal(cur.ExcludePaths, cfg.ExcludePaths) +} + func (m *matrixSource) setConfig(cfg config.MatrixConfig) { + if m.samePolicy(cfg) { + return + } m.policyMu.Lock() defer m.policyMu.Unlock() + if m.samePolicy(cfg) { + return + } c := cfg c.Repos = append([]string(nil), cfg.Repos...) c.ExcludePaths = append([]string(nil), cfg.ExcludePaths...) @@ -188,6 +204,11 @@ func (m *matrixSource) run() { return case queued := <-m.in: if queued.epoch != m.checkpoints.policyEpoch() { + // Queued under a policy that no longer applies: neither + // projection ingests it. setConfig already reset the + // legacy backlog for the same change, so admitting the + // stale observation there would leak code the new policy + // never saw. continue } before := m.Latest() From 3338fc2537d4a51dfbd9835fb260a95e5c9bb88f Mon Sep 17 00:00:00 2001 From: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> Date: Wed, 4 Nov 2026 02:18:15 -0500 Subject: [PATCH 6/6] =?UTF-8?q?=E2=99=BB=EF=B8=8F=20refactor(dashboard):?= =?UTF-8?q?=20drop=20the=20Matrix=20persist=20stub=20and=20name=20checkpoi?= =?UTF-8?q?nt=20labels?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Address review findings on #16513: - changelog.d/fixed-matrix-ingest-freshness.md no longer promises /data/matrix-lines.json persistence, which this PR removes. - Re-attach the matrixSource doc paragraph to matrixSource. - Delete the no-op SetMatrixPersistPath stub and matrix_persist_test.go; the legacy-history-never-restored assertion moves into TestCheckpointMemoryOnly. - Name the "Pushed PR" / "pushed-checkpoint" response contract as constants shared by ingest and since(). Hive-Fix-Attempt: 4/6 Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> Signed-off-by: scanner <280983584+hivecommons-hive[bot]@users.noreply.github.com> --- changelog.d/fixed-matrix-ingest-freshness.md | 2 +- src/pkg/dashboard/checkpoint_source.go | 11 ++++-- src/pkg/dashboard/checkpoint_source_test.go | 25 ++++++++++++-- src/pkg/dashboard/matrix_persist_test.go | 36 -------------------- src/pkg/dashboard/matrix_source.go | 13 +++---- 5 files changed, 37 insertions(+), 50 deletions(-) delete mode 100644 src/pkg/dashboard/matrix_persist_test.go diff --git a/changelog.d/fixed-matrix-ingest-freshness.md b/changelog.d/fixed-matrix-ingest-freshness.md index fdb1d8b350..639a4a819c 100644 --- a/changelog.d/fixed-matrix-ingest-freshness.md +++ b/changelog.d/fixed-matrix-ingest-freshness.md @@ -1 +1 @@ -- Matrix kiosk no longer goes stale for the length of a governor tick: an incremental sentinel pass now reads only new PR heads in the allowlisted repos every two minutes, independently of the full 15-minute sweep, and `/api/matrix/lines` reports `last_ingest_at` / `next_ingest_in_s` so the footer can tell "nothing pushed" from "nothing ingested". The matrix backlog (last 500 lines) and the set of already-evaluated PR heads now persist under `/data/matrix-lines.json` and `/data/sentinel-heads.json` and are restored at boot with the sequence counter continuing, so a hub-triggered restart no longer blanks the kiosk or re-diffs every open PR; the first ingest also runs at boot instead of waiting for the reconcile timer ([#16276](https://github.com/hivecommons/hive/issues/16276)). +- Matrix kiosk no longer goes stale for the length of a governor tick: an incremental sentinel pass now reads only new PR heads in the allowlisted repos every two minutes, independently of the full 15-minute sweep, and `/api/matrix/lines` reports `last_ingest_at` / `next_ingest_in_s` so the footer can tell "nothing pushed" from "nothing ingested". The set of already-evaluated PR heads now persists under `/data/sentinel-heads.json` and is restored at boot, so a hub-triggered restart no longer re-diffs every open PR; the first ingest also runs at boot instead of waiting for the reconcile timer ([#16276](https://github.com/hivecommons/hive/issues/16276)). diff --git a/src/pkg/dashboard/checkpoint_source.go b/src/pkg/dashboard/checkpoint_source.go index a401d790c8..5fd0efa295 100644 --- a/src/pkg/dashboard/checkpoint_source.go +++ b/src/pkg/dashboard/checkpoint_source.go @@ -25,6 +25,13 @@ const ( checkpointMaxBytes = 512 << 10 ) +// Response contract shared by ingest and since(): the reader shows +// checkpointSourceLabel as the feed source and checkpointStage on every region. +const ( + checkpointSourceLabel = "Pushed PR" + checkpointStage = "pushed-checkpoint" +) + type checkpointLine struct { Op string `json:"op"` Line string `json:"line"` @@ -148,7 +155,7 @@ func (c *checkpointSource) ingest(obs github.SentinelPatchObservation, epoch str continue } c.seq++ - region := checkpointRegion{Seq: c.seq, Repo: obs.Repo, PR: obs.Number, PRURL: fmt.Sprintf("https://github.com/%s/pull/%d", obs.Repo, obs.Number), Head: obs.HeadSHA, Path: file.Path, Stage: "pushed-checkpoint", At: at, Lines: lines} + region := checkpointRegion{Seq: c.seq, Repo: obs.Repo, PR: obs.Number, PRURL: fmt.Sprintf("https://github.com/%s/pull/%d", obs.Repo, obs.Number), Head: obs.HeadSHA, Path: file.Path, Stage: checkpointStage, At: at, Lines: lines} for len(c.regions) > 0 && (c.lines+len(lines) > matrixBacklogCap || c.bytes+size > checkpointMaxBytes) { old := c.regions[0] c.lines -= len(old.Lines) @@ -288,7 +295,7 @@ func (c *checkpointSource) since(seq uint64, limit int) checkpointResponse { if seq > c.seq { seq = 0 } - out := checkpointResponse{Seq: seq, Latest: c.seq, Epoch: c.epoch, Source: "Pushed PR", Stage: "pushed-checkpoint", Regions: []checkpointRegion{}} + out := checkpointResponse{Seq: seq, Latest: c.seq, Epoch: c.epoch, Source: checkpointSourceLabel, Stage: checkpointStage, Regions: []checkpointRegion{}} count := 0 for _, r := range c.regions { if r.Seq <= seq { diff --git a/src/pkg/dashboard/checkpoint_source_test.go b/src/pkg/dashboard/checkpoint_source_test.go index c149c56c14..b2d77090d5 100644 --- a/src/pkg/dashboard/checkpoint_source_test.go +++ b/src/pkg/dashboard/checkpoint_source_test.go @@ -7,6 +7,8 @@ import ( "log/slog" "net/http" "net/http/httptest" + "os" + "path/filepath" "reflect" "strings" "testing" @@ -35,7 +37,7 @@ func TestCheckpointWholeRegionsAndContinuation(t *testing.T) { } want := []checkpointLine{{"-", "old"}, {"+", "\tnew"}, {"+", ""}, {"+", "repeat"}, {"+", "repeat"}} r := first.Regions[0] - if !reflect.DeepEqual(r.Lines, want) || r.Head != "real-head" || r.PRURL != "https://github.com/"+matrixTestRepo+"/pull/3" || r.Stage != "pushed-checkpoint" || !r.At.Equal(now) { + if !reflect.DeepEqual(r.Lines, want) || r.Head != "real-head" || r.PRURL != "https://github.com/"+matrixTestRepo+"/pull/3" || r.Stage != checkpointStage || !r.At.Equal(now) { t.Fatalf("region %+v", r) } next := c.since(first.Seq, 1) @@ -285,7 +287,7 @@ func TestCheckpointAPIAuthColdAndEpoch(t *testing.T) { } rr := request("Bearer checkpoint-test-token", "") var cold checkpointResponse - if rr.Code != 200 || json.Unmarshal(rr.Body.Bytes(), &cold) != nil || cold.Epoch == "" || cold.LastIngestAt != nil || len(cold.Regions) != 0 || cold.Source != "Pushed PR" || cold.Stage != "pushed-checkpoint" { + if rr.Code != 200 || json.Unmarshal(rr.Body.Bytes(), &cold) != nil || cold.Epoch == "" || cold.LastIngestAt != nil || len(cold.Regions) != 0 || cold.Source != checkpointSourceLabel || cold.Stage != checkpointStage { t.Fatalf("cold response %d %s", rr.Code, rr.Body.String()) } // Feed the real observer queue, including a deletion-only checkpoint. @@ -339,6 +341,25 @@ func TestCheckpointMemoryOnly(t *testing.T) { if len(restored.checkpoints.since(0, 200).Regions) != 0 || restored.checkpoints.policyEpoch() == m.checkpoints.policyEpoch() { t.Fatal("new source restored checkpoint history") } + // The superseded persistent line history must never re-enter the feed and + // is left untouched for the operator to remove, rather than silently + // deleted or rewritten as part of a dashboard upgrade. + path := filepath.Join(t.TempDir(), "matrix-lines.json") + seed := []byte(`{"seq":7,"lines":[{"seq":7,"repo":"org/repo","pr":1,"line":"old retained code"}]}`) + if err := os.WriteFile(path, seed, 0600); err != nil { + t.Fatal(err) + } + s := &Server{} + observe := s.MatrixObserver(matrixTestConfig()) + defer func() { s.matrix.close(); <-s.matrix.done }() + if s.matrix.Latest() != 0 || len(s.matrix.checkpoints.since(0, 200).Regions) != 0 { + t.Fatal("legacy persistent code restored") + } + observe(matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: checkpointPatch("fresh checkpoint")})) + waitFor(t, func() bool { return s.matrix.Latest() != 0 }, "observer ingest") + if got, err := os.ReadFile(path); err != nil || string(got) != string(seed) { + t.Fatal("legacy history file changed") + } // Compile-time assertion: the public observer still receives the existing // authoritative transport, with no extra patch collector or endpoint. var _ func(github.SentinelPatchObservation) = m.Observe diff --git a/src/pkg/dashboard/matrix_persist_test.go b/src/pkg/dashboard/matrix_persist_test.go deleted file mode 100644 index 58aeb72b8d..0000000000 --- a/src/pkg/dashboard/matrix_persist_test.go +++ /dev/null @@ -1,36 +0,0 @@ -package dashboard - -import ( - "os" - "path/filepath" - "testing" - - "github.com/hivecommons/hive/pkg/sentinel" -) - -// The superseded persistent line history must never re-enter either feed. -// Existing files are left for the operator to remove, rather than silently -// deleting their data as part of a dashboard config or software upgrade. -func TestMatrixCodeHistoryIsMemoryOnly(t *testing.T) { - path := filepath.Join(t.TempDir(), "matrix-lines.json") - seed := []byte(`{"seq":7,"lines":[{"seq":7,"repo":"org/repo","pr":1,"line":"old retained code"}]}`) - if err := os.WriteFile(path, seed, 0600); err != nil { - t.Fatal(err) - } - s := &Server{} - s.SetMatrixPersistPath(path) - observe := s.MatrixObserver(matrixTestConfig()) - defer func() { s.matrix.close(); <-s.matrix.done }() - if s.matrix.Latest() != 0 || len(s.matrix.checkpoints.since(0, 200).Regions) != 0 { - t.Fatal("legacy persistent code restored") - } - observe(matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: checkpointPatch("fresh checkpoint")})) - waitFor(t, func() bool { return s.matrix.Latest() != 0 }, "observer ingest") - s.SetMatrixPersistPath(path) - got, err := os.ReadFile(path) - if err != nil || string(got) != string(seed) { - t.Fatal("legacy history file changed") - } - var nilServer *Server - nilServer.SetMatrixPersistPath(path) -} diff --git a/src/pkg/dashboard/matrix_source.go b/src/pkg/dashboard/matrix_source.go index 1d23c6c005..2a891a285c 100644 --- a/src/pkg/dashboard/matrix_source.go +++ b/src/pkg/dashboard/matrix_source.go @@ -52,15 +52,15 @@ type matrixPRState struct { touched time.Time } -// matrixSource turns sentinel patch observations into a bounded, scrubbed, -// de-duplicated in-memory backlog. Observe never blocks; one goroutine does -// all the parsing. Nothing is written to disk. type matrixQueuedObservation struct { observation github.SentinelPatchObservation epoch string } -// matrixSource owns the shared observer queue and its legacy line projection. +// matrixSource turns sentinel patch observations into a bounded, scrubbed, +// de-duplicated in-memory backlog. It owns the shared observer queue and its +// legacy line projection. Observe never blocks; one goroutine does all the +// parsing. Nothing is written to disk. type matrixSource struct { checkpoints *checkpointSource // policyMu is held exclusively by setConfig and shared by observers, so @@ -501,8 +501,3 @@ func (s *Server) RecordMatrixIngest(at time.Time, every time.Duration) { } m.recordIngest(at, every) } - -// SetMatrixPersistPath is retained for callers compiled against the Matrix -// prototype. Code activity is memory-only: no legacy file is read or written. -// Deprecated: persistent code history is no longer supported. -func (s *Server) SetMatrixPersistPath(_ string) {}