diff --git a/changelog.d/added-matrix-line-source.md b/changelog.d/added-matrix-line-source.md new file mode 100644 index 0000000000..0bb4b225a0 --- /dev/null +++ b/changelog.d/added-matrix-line-source.md @@ -0,0 +1 @@ +- **Activity Monitor Matrix: bounded, scrub-drop line source behind `dashboard.matrix`** (#15934, part of #15862). When `dashboard.matrix` is enabled with an allowlist, the sentinel's existing patch fetch now feeds an in-memory source of pushed lines — no new GitHub calls, nothing written to disk. Only hive-identified authors on public, allowlisted repos are admitted; secret-shaped and generated paths are skipped before parsing; any line that `logscrub` would change, that already carries a redaction marker, that looks high-entropy or exceeds 160 runes is dropped rather than masked. Ingest is non-blocking (a full 64-slot channel drops and counts), with dedup per PR, 200 lines per head, a 2,000-line backlog and a per-minute rate cap. Default off: with the flag off no observer is registered and nothing is retained. No HTTP surface yet; the endpoint follows in #15935. diff --git a/src/cmd/hive/main_sweeps.go b/src/cmd/hive/main_sweeps.go index dc4516166c..9daf317bee 100644 --- a/src/cmd/hive/main_sweeps.go +++ b/src/cmd/hive/main_sweeps.go @@ -1286,7 +1286,14 @@ const sentinelSweepInterval = github.SentinelReconcileInterval // trust or rules. func sentinelSweepOptions(cfg *config.Config, ghClient *github.Client, dashSrv *dashboard.Server) github.SentinelSweepOptions { sc := cfg.Sentinel + // The Matrix line source only ever sees patches when dashboard.matrix is + // active (#15934); otherwise PatchObserver stays nil and nothing is kept. + var patchObserver func(github.SentinelPatchObservation) + if dashSrv != nil { + patchObserver = dashSrv.MatrixObserver(cfg.Dashboard.Matrix) + } return github.SentinelSweepOptions{ + PatchObserver: patchObserver, Label: sc.LabelOrDefault(), LabelColor: config.DefaultSentinelLabelColor, LabelDescription: config.DefaultSentinelLabelDescription, diff --git a/src/cmd/hive/main_sweeps_matrix_test.go b/src/cmd/hive/main_sweeps_matrix_test.go new file mode 100644 index 0000000000..cb9bdd895e --- /dev/null +++ b/src/cmd/hive/main_sweeps_matrix_test.go @@ -0,0 +1,43 @@ +package main + +import ( + "io" + "log/slog" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/dashboard" + hgithub "github.com/hivecommons/hive/pkg/github" +) + +// The Matrix line source must only see sentinel patches when +// dashboard.matrix is active (#15934). +func TestSentinelSweepOptionsMatrixObserverWiring(t *testing.T) { + ghClient := hgithub.NewClient("", "hivecommons", []string{"hivecommons/hive"}, nil, "") + tests := []struct { + name string + matrix config.MatrixConfig + withSrv bool + want bool + }{ + {name: "default off", withSrv: true}, + {name: "enabled without allowlist", matrix: config.MatrixConfig{Enabled: true}, withSrv: true}, + {name: "allowlist without enabled", matrix: config.MatrixConfig{Repos: []string{"hivecommons/hive"}}, withSrv: true}, + {name: "active without dashboard", matrix: config.MatrixConfig{Enabled: true, Repos: []string{"hivecommons/hive"}}}, + {name: "active", matrix: config.MatrixConfig{Enabled: true, Repos: []string{"hivecommons/hive"}}, withSrv: true, want: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + cfg := &config.Config{} + cfg.Dashboard.Matrix = tc.matrix + var dashSrv *dashboard.Server + if tc.withSrv { + dashSrv = dashboard.NewServer(0, slog.New(slog.NewTextHandler(io.Discard, nil))) + } + opts := sentinelSweepOptions(cfg, ghClient, dashSrv) + if got := opts.PatchObserver != nil; got != tc.want { + t.Fatalf("PatchObserver set = %v, want %v", got, tc.want) + } + }) + } +} diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 46164e71cf..bd398f6d7a 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1651` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1652` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1653` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1656` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1657` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1658` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index b173e765e2..59e1fe61a8 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -64,4 +64,4 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:337` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:338` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:339` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1658` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1663` | diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index c43474b235..4a4f48d6e6 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:44` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:45` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:69` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1631` | -| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1632` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1633` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1639` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`) | `pkg/dashboard/server.go:1644` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1645` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1646` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1647` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1648` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1649` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1650` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1636` | +| `GET` | `/api/health/deep` | Public | Deep health probe | `pkg/dashboard/server.go:1637` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1638` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1644` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`) | `pkg/dashboard/server.go:1649` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1650` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1651` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1652` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1653` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1654` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1655` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 52e97c8a85..95b6ccd810 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:46` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1711` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1716` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:82` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:83` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:84` | diff --git a/src/internal/testutil/dashboard_import_allowlist.txt b/src/internal/testutil/dashboard_import_allowlist.txt index 7598aa273d..ca6a37199c 100644 --- a/src/internal/testutil/dashboard_import_allowlist.txt +++ b/src/internal/testutil/dashboard_import_allowlist.txt @@ -11,6 +11,7 @@ github.com/hivecommons/hive/crux github.com/hivecommons/hive/crux/agentparse github.com/hivecommons/hive/crux/claude github.com/hivecommons/hive/crux/kubejob +github.com/hivecommons/hive/crux/logscrub github.com/hivecommons/hive/crux/paneclass github.com/hivecommons/hive/crux/sandbox github.com/hivecommons/hive/crux/skillreg diff --git a/src/pkg/dashboard/matrix_source.go b/src/pkg/dashboard/matrix_source.go new file mode 100644 index 0000000000..323515e468 --- /dev/null +++ b/src/pkg/dashboard/matrix_source.go @@ -0,0 +1,360 @@ +package dashboard + +import ( + "crypto/sha256" + "fmt" + "log/slog" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" +) + +// Matrix line source bounds (#15934). Every buffer is fixed so a burst of +// pushes can cost memory and CPU only up to these limits. +const ( + matrixIngestCap = 64 + matrixMaxLineRunes = 160 + matrixMaxLinesPerHead = 200 + matrixBacklogCap = 2000 + matrixMaxTrackedPRs = 500 + matrixDedupTTL = 24 * time.Hour + matrixRateWindow = time.Minute + matrixMaxLinesPerRate = 600 + matrixMaxFilesPerObs = 300 + matrixMaxPatchBytes = 512 << 10 + matrixDefaultSinceMax = 200 +) + +// matrixMaxDedupHashes caps dedup hashes across every tracked PR; a var so +// tests can exercise the bound without 100k lines. +var matrixMaxDedupHashes = 100000 + +// matrixLine is one accepted, scrub-clean added line. +type matrixLine struct { + Seq uint64 `json:"seq"` + Repo string `json:"repo"` + PR int `json:"pr"` + Head string `json:"head"` + Path string `json:"path"` + Lang string `json:"lang,omitempty"` + Line string `json:"line"` + At time.Time `json:"at"` +} + +type matrixPRState struct { + head string + headCount int + hashes map[[sha256.Size]byte]time.Time + touched time.Time +} + +// matrixSource turns sentinel patch observations into a bounded, scrubbed, +// de-duplicated in-memory backlog. Observe never blocks; one goroutine does +// all the parsing. Nothing is written to disk. +type matrixSource struct { + cfg atomic.Pointer[config.MatrixConfig] + in chan github.SentinelPatchObservation + dropped atomic.Uint64 + logger *slog.Logger + now func() time.Time + + startOnce sync.Once + stopOnce sync.Once + stop chan struct{} + done chan struct{} + + mu sync.Mutex + ring []matrixLine + next int + seq uint64 + prs map[string]*matrixPRState + totalHashes int + rateStart time.Time + rateCount int + rateDropped uint64 + notify func(seq uint64) +} + +func newMatrixSource(cfg config.MatrixConfig, logger *slog.Logger) *matrixSource { + if logger == nil { + logger = slog.Default() + } + m := &matrixSource{ + in: make(chan github.SentinelPatchObservation, matrixIngestCap), + logger: logger, + now: time.Now, + stop: make(chan struct{}), + done: make(chan struct{}), + ring: make([]matrixLine, 0, matrixBacklogCap), + prs: make(map[string]*matrixPRState), + } + m.setConfig(cfg) + return m +} + +func (m *matrixSource) setConfig(cfg config.MatrixConfig) { + c := cfg + c.Repos = append([]string(nil), cfg.Repos...) + c.ExcludePaths = append([]string(nil), cfg.ExcludePaths...) + m.cfg.Store(&c) +} + +// admits applies the observation-level gates: Matrix active, hive-identified +// author, public repo on the explicit allowlist. There is no way to widen it. +func (m *matrixSource) admits(obs github.SentinelPatchObservation) bool { + cfg := m.cfg.Load() + return cfg != nil && cfg.Active() && obs.HiveIdentified && !obs.Private && cfg.RepoAllowed(obs.Repo) +} + +// Observe is the sentinel PatchObserver. It gates the observation, then hands +// it off without blocking: when the ingest channel is full the observation +// is dropped and counted. +func (m *matrixSource) Observe(obs github.SentinelPatchObservation) { + if !m.admits(obs) { + return + } + select { + case m.in <- obs: + default: + m.dropped.Add(1) + } +} + +// Dropped counts observations Observe discarded because the channel was full. +func (m *matrixSource) Dropped() uint64 { return m.dropped.Load() } + +func (m *matrixSource) start() { + m.startOnce.Do(func() { go m.run() }) +} + +func (m *matrixSource) close() { + m.stopOnce.Do(func() { close(m.stop) }) +} + +func (m *matrixSource) run() { + defer close(m.done) + for { + select { + case <-m.stop: + return + case obs := <-m.in: + m.safeIngest(obs) + } + } +} + +func (m *matrixSource) safeIngest(obs github.SentinelPatchObservation) { + defer func() { + if r := recover(); r != nil { + m.logger.Warn("matrix line source dropped an observation after a panic", "repo", obs.Repo, "number", obs.Number, "panic", fmt.Sprint(r)) + } + }() + m.ingest(obs) +} + +// ingest filters one observation into the backlog and returns how many lines +// it added. +func (m *matrixSource) ingest(obs github.SentinelPatchObservation) int { + if !m.admits(obs) { + return 0 + } + excludes := m.cfg.Load().ExcludePaths + now := m.now() + at := obs.At + if at.IsZero() { + at = now + } + key := obs.Repo + "#" + strconv.Itoa(obs.Number) + + m.mu.Lock() + m.pruneLocked(now) + st := m.prs[key] + if st == nil { + st = &matrixPRState{hashes: make(map[[sha256.Size]byte]time.Time)} + m.prs[key] = st + } + st.touched = now + if st.head != obs.HeadSHA { + st.head = obs.HeadSHA + st.headCount = 0 + } + added := 0 + files := obs.Files + if len(files) > matrixMaxFilesPerObs { + files = files[:matrixMaxFilesPerObs] + } +scan: + for _, f := range files { + if matrixPathDenied(f.Path, excludes) || len(f.Patch) > matrixMaxPatchBytes { + continue + } + lang := matrixLang(f.Path) + for _, raw := range matrixAddedLines(f.Patch) { + if st.headCount >= matrixMaxLinesPerHead { + break scan + } + line, ok := matrixAcceptLine(raw) + if !ok { + continue + } + h := sha256.Sum256([]byte(f.Path + "\x00" + line)) + if seen, dup := st.hashes[h]; dup && now.Sub(seen) < matrixDedupTTL { + continue + } + if !m.rateAllowLocked(now) { + break scan + } + if _, dup := st.hashes[h]; !dup { + m.totalHashes++ + } + st.hashes[h] = now + st.headCount++ + m.seq++ + m.appendLocked(matrixLine{Seq: m.seq, Repo: obs.Repo, PR: obs.Number, Head: obs.HeadSHA, Path: f.Path, Lang: lang, Line: line, At: at}) + added++ + } + } + m.evictLocked(key) + seq, notify := m.seq, m.notify + m.mu.Unlock() + + if added > 0 && notify != nil { + notify(seq) + } + return added +} + +func (m *matrixSource) appendLocked(l matrixLine) { + if len(m.ring) < matrixBacklogCap { + m.ring = append(m.ring, l) + return + } + m.ring[m.next] = l + m.next = (m.next + 1) % matrixBacklogCap +} + +// rateAllowLocked admits at most matrixMaxLinesPerRate lines per window. +func (m *matrixSource) rateAllowLocked(now time.Time) bool { + if now.Sub(m.rateStart) >= matrixRateWindow || now.Before(m.rateStart) { + m.rateStart = now + m.rateCount = 0 + } + if m.rateCount >= matrixMaxLinesPerRate { + m.rateDropped++ + return false + } + m.rateCount++ + return true +} + +// pruneLocked forgets PR sets untouched for a full dedup TTL. +func (m *matrixSource) pruneLocked(now time.Time) { + for key, st := range m.prs { + if now.Sub(st.touched) >= matrixDedupTTL { + m.removeLocked(key) + } + } +} + +// evictLocked drops the least recently touched PR sets until both the PR +// count and the global hash count are within bounds. keep is the PR just +// ingested; it goes last. +func (m *matrixSource) evictLocked(keep string) { + for len(m.prs) > matrixMaxTrackedPRs || m.totalHashes > matrixMaxDedupHashes { + oldest := "" + var oldestAt time.Time + for key, st := range m.prs { + if key == keep && len(m.prs) > 1 { + continue + } + if oldest == "" || st.touched.Before(oldestAt) { + oldest, oldestAt = key, st.touched + } + } + if oldest == "" { + return + } + m.removeLocked(oldest) + } +} + +func (m *matrixSource) removeLocked(key string) { + if st, ok := m.prs[key]; ok { + m.totalHashes -= len(st.hashes) + delete(m.prs, key) + } +} + +// Since returns up to limit backlog lines with Seq > seq, oldest first, and the +// latest sequence number. limit <= 0 means a default page. +func (m *matrixSource) Since(seq uint64, limit int) ([]matrixLine, uint64) { + if limit <= 0 { + limit = matrixDefaultSinceMax + } + m.mu.Lock() + defer m.mu.Unlock() + n := len(m.ring) + out := make([]matrixLine, 0, min(limit, n)) + for i := 0; i < n && len(out) < limit; i++ { + l := m.ring[(m.next+i)%n] + if l.Seq > seq { + out = append(out, l) + } + } + return out, m.seq +} + +// Latest is the sequence number of the newest accepted line; 0 when none. +func (m *matrixSource) Latest() uint64 { + m.mu.Lock() + defer m.mu.Unlock() + return m.seq +} + +// reset forgets the backlog and dedup state; Seq keeps counting so a reader +// never sees a sequence number reused. +func (m *matrixSource) reset() { + m.mu.Lock() + m.ring = m.ring[:0] + m.next = 0 + m.prs = make(map[string]*matrixPRState) + m.totalHashes = 0 + m.mu.Unlock() +} + +func (m *matrixSource) setNotify(fn func(seq uint64)) { + m.mu.Lock() + m.notify = fn + m.mu.Unlock() +} + +// MatrixObserver returns the sentinel PatchObserver feeding the Matrix line +// source, or nil when dashboard.matrix is not active — then nothing is +// created, and a source left over from an earlier +// active config stops admitting and forgets its backlog. The source is built on first use and +// later calls refresh its allowlist and exclude globs. +func (s *Server) MatrixObserver(cfg config.MatrixConfig) func(github.SentinelPatchObservation) { + if s == nil { + return nil + } + s.matrixMu.Lock() + defer s.matrixMu.Unlock() + if !cfg.Active() { + if s.matrix != nil { + s.matrix.setConfig(cfg) + s.matrix.reset() + } + return nil + } + if s.matrix == nil { + s.matrix = newMatrixSource(cfg, s.logger) + s.matrix.start() + } else { + s.matrix.setConfig(cfg) + } + return s.matrix.Observe +} diff --git a/src/pkg/dashboard/matrix_source_filter.go b/src/pkg/dashboard/matrix_source_filter.go new file mode 100644 index 0000000000..b1e49b9d0e --- /dev/null +++ b/src/pkg/dashboard/matrix_source_filter.go @@ -0,0 +1,188 @@ +package dashboard + +import ( + "path" + "strings" + "unicode" + "unicode/utf8" + + "github.com/hivecommons/hive/crux/logscrub" +) + +// matrixEntropyRun is the longest run of token-alphabet characters a Matrix +// line may carry; a longer run looks like a key, hash or encoded secret. +const matrixEntropyRun = 32 + +var matrixLockfiles = map[string]bool{ + "go.sum": true, + "package-lock.json": true, + "npm-shrinkwrap.json": true, + "yarn.lock": true, + "pnpm-lock.yaml": true, + "cargo.lock": true, + "gemfile.lock": true, + "poetry.lock": true, + "pipfile.lock": true, + "composer.lock": true, + "bun.lockb": true, +} + +// matrixPathDenied reports whether a file must be skipped before its patch is +// parsed: secret-shaped names, lockfiles, vendored or generated code, and the +// operator's dashboard.matrix.exclude_paths globs. +func matrixPathDenied(p string, excludes []string) bool { + if strings.TrimSpace(p) == "" { + return true + } + lower := strings.ToLower(p) + base := path.Base(lower) + switch { + case strings.HasPrefix(base, ".env"), + strings.HasSuffix(base, ".pem"), + strings.HasSuffix(base, ".key"), + strings.HasSuffix(base, ".p12"), + strings.HasPrefix(base, "id_"), + matrixLockfiles[base], + strings.HasSuffix(base, ".lock"), + strings.Contains(base, ".min."), + strings.Contains(lower, "zz_generated"), + strings.HasSuffix(base, ".pb.go"), + strings.Contains(base, "_gen."): + return true + } + segments := strings.Split(lower, "/") + for i, seg := range segments { + if strings.Contains(seg, "secret") || strings.Contains(seg, "credential") || strings.Contains(seg, "token") { + return true + } + if i < len(segments)-1 && (seg == "vendor" || seg == "node_modules") { + return true + } + } + for _, glob := range excludes { + glob = strings.TrimSpace(glob) + if glob == "" { + continue + } + if ok, _ := path.Match(glob, p); ok { + return true + } + if ok, _ := path.Match(glob, path.Base(p)); ok { + return true + } + if dir := strings.TrimSuffix(glob, "/**"); dir != glob && (p == dir || strings.HasPrefix(p, dir+"/")) { + return true + } + } + return false +} + +// matrixAddedLines returns the candidate lines a patch added: `+` lines +// without the `+++` header, minus anything inside a BEGIN/END armour block +// (a PEM key is only recognisable as a whole, never line by line). A patch +// that is empty or carries NUL bytes is treated as binary and yields nothing. +func matrixAddedLines(patch string) []string { + if patch == "" || strings.ContainsRune(patch, 0) { + return nil + } + var out []string + inArmour := false + for _, raw := range strings.Split(patch, "\n") { + if strings.HasPrefix(raw, "+++") || !strings.HasPrefix(raw, "+") { + continue + } + line := raw[1:] + if strings.Contains(line, "-----BEGIN ") { + inArmour = true + } + if inArmour { + if strings.Contains(line, "-----END ") { + inArmour = false + } + continue + } + out = append(out, line) + } + return out +} + +// matrixAcceptLine trims a candidate line and reports whether it may be +// shown. Every rule drops; nothing is ever masked, so a rejected line never +// leaves a partial secret behind. +func matrixAcceptLine(raw string) (string, bool) { + line := strings.TrimSpace(raw) + if line == "" || !utf8.ValidString(line) { + return "", false + } + if utf8.RuneCountInString(line) > matrixMaxLineRunes { + return "", false + } + for _, r := range line { + if unicode.IsControl(r) && r != '\t' { + return "", false + } + } + if matrixHasEntropyRun(line) { + return "", false + } + if logscrub.ContainsRedactionMarker(line) { + return "", false + } + if logscrub.ScrubString(line) != line || logscrub.RedactPaneTokens(line) != line { + return "", false + } + return line, true +} + +func matrixHasEntropyRun(s string) bool { + run := 0 + for i := 0; i < len(s); i++ { + c := s[i] + if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') || + c == '+' || c == '/' || c == '=' || c == '_' || c == '-' { + run++ + if run >= matrixEntropyRun { + return true + } + continue + } + run = 0 + } + return false +} + +var matrixLangByExt = map[string]string{ + ".go": "go", + ".js": "javascript", + ".mjs": "javascript", + ".cjs": "javascript", + ".jsx": "javascript", + ".ts": "typescript", + ".tsx": "typescript", + ".py": "python", + ".rb": "ruby", + ".rs": "rust", + ".java": "java", + ".kt": "kotlin", + ".c": "c", + ".h": "c", + ".cc": "cpp", + ".cpp": "cpp", + ".hpp": "cpp", + ".cs": "csharp", + ".sh": "shell", + ".bash": "shell", + ".yaml": "yaml", + ".yml": "yaml", + ".json": "json", + ".toml": "toml", + ".md": "markdown", + ".html": "html", + ".css": "css", + ".sql": "sql", +} + +// matrixLang names a file's language from its extension; "" when unknown. +func matrixLang(p string) string { + return matrixLangByExt[strings.ToLower(path.Ext(p))] +} diff --git a/src/pkg/dashboard/matrix_source_test.go b/src/pkg/dashboard/matrix_source_test.go new file mode 100644 index 0000000000..37c510f69a --- /dev/null +++ b/src/pkg/dashboard/matrix_source_test.go @@ -0,0 +1,483 @@ +package dashboard + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/github" + "github.com/hivecommons/hive/pkg/sentinel" +) + +const matrixTestRepo = "hivecommons/hive" + +func matrixTestConfig() config.MatrixConfig { + return config.MatrixConfig{Enabled: true, Repos: []string{matrixTestRepo}} +} + +type matrixClock struct{ t time.Time } + +func (c *matrixClock) now() time.Time { return c.t } +func (c *matrixClock) advance(d time.Duration) { c.t = c.t.Add(d) } + +func newTestMatrixSource(t *testing.T, cfg config.MatrixConfig) (*matrixSource, *matrixClock) { + t.Helper() + m := newMatrixSource(cfg, nil) + clk := &matrixClock{t: time.Date(2026, 10, 1, 12, 0, 0, 0, time.UTC)} + m.now = clk.now + return m, clk +} + +func matrixObs(number int, head string, files ...sentinel.File) github.SentinelPatchObservation { + return github.SentinelPatchObservation{ + Repo: matrixTestRepo, + Number: number, + HeadSHA: head, + Author: "hive-bot", + HiveIdentified: true, + Files: files, + } +} + +func matrixPatch(lines ...string) string { + var b strings.Builder + b.WriteString("--- a/x\n+++ b/x\n@@ -0,0 +1 @@\n") + for _, l := range lines { + b.WriteString("+" + l + "\n") + } + return b.String() +} + +func matrixUniqueLines(prefix string, n int) []string { + out := make([]string, n) + for i := range out { + out[i] = fmt.Sprintf("%s line %d", prefix, i) + } + return out +} + +func matrixAllLines(m *matrixSource) []matrixLine { + lines, _ := m.Since(0, matrixBacklogCap+1) + return lines +} + +func TestMatrixSourceDropsSecretsFromFixturePatch(t *testing.T) { + m, _ := newTestMatrixSource(t, matrixTestConfig()) + ghToken := "ghp_" + strings.Repeat("a1B2", 9) + awsKey := "AKIA" + "IOSFODNN7EXAMPLE" + scrubbed := "aws_key = " + awsKey + paneToken := "key: sk-" + "abc123def" + files := []sentinel.File{ + {Path: "pkg/client.go", Patch: matrixPatch( + "func main() {", + "token := \""+ghToken+"\"", + scrubbed, + paneToken, + "note: [REDACTED] earlier", + "-----BEGIN RSA PRIVATE KEY-----", + "MIIEowIBAAKCAQEA", + "abc", + "-----END RSA PRIVATE KEY-----", + "return nil", + )}, + {Path: "certs/server.pem", Patch: matrixPatch("pem visible line")}, + {Path: ".env", Patch: matrixPatch("DB_HOST=localhost")}, + {Path: "deploy/secrets.yaml", Patch: matrixPatch("password: hunter2")}, + } + if got := m.ingest(matrixObs(1, "h1", files...)); got != 2 { + t.Fatalf("ingest added %d lines, want 2", got) + } + var got []string + for _, l := range matrixAllLines(m) { + got = append(got, l.Line) + if l.Repo != matrixTestRepo || l.PR != 1 || l.Head != "h1" || l.Path != "pkg/client.go" || l.Lang != "go" || l.At.IsZero() { + t.Errorf("unexpected line metadata: %+v", l) + } + } + if strings.Join(got, "|") != "func main() {|return nil" { + t.Fatalf("accepted lines = %q", got) + } + for _, l := range got { + for _, bad := range []string{ghToken, awsKey, "sk-", "REDACTED", "PRIVATE KEY", "MIIE", "abc", "pem visible", "DB_HOST", "hunter2"} { + if strings.Contains(l, bad) { + t.Errorf("line %q leaks %q", l, bad) + } + } + } +} + +func TestMatrixSourceObservationGates(t *testing.T) { + file := sentinel.File{Path: "main.go", Patch: matrixPatch("x := 1")} + tests := []struct { + name string + cfg config.MatrixConfig + mutate func(*github.SentinelPatchObservation) + want int + }{ + {name: "accepted", cfg: matrixTestConfig(), want: 1}, + {name: "not hive identified", cfg: matrixTestConfig(), mutate: func(o *github.SentinelPatchObservation) { o.HiveIdentified = false }}, + {name: "private repo", cfg: matrixTestConfig(), mutate: func(o *github.SentinelPatchObservation) { o.Private = true }}, + {name: "repo not allowlisted", cfg: matrixTestConfig(), mutate: func(o *github.SentinelPatchObservation) { o.Repo = "other/repo" }}, + {name: "disabled", cfg: config.MatrixConfig{Repos: []string{matrixTestRepo}}}, + {name: "empty allowlist", cfg: config.MatrixConfig{Enabled: true}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m, _ := newTestMatrixSource(t, tc.cfg) + obs := matrixObs(7, "h", file) + if tc.mutate != nil { + tc.mutate(&obs) + } + m.Observe(obs) + if tc.want == 0 && len(m.in) != 0 { + t.Fatalf("Observe queued a rejected observation") + } + if got := m.ingest(obs); got != tc.want { + t.Fatalf("ingest = %d, want %d", got, tc.want) + } + if got := len(matrixAllLines(m)); got != tc.want { + t.Fatalf("backlog = %d, want %d", got, tc.want) + } + }) + } +} + +func TestMatrixPathDenied(t *testing.T) { + excludes := []string{"docs/*.md", "*.snap", "fixtures/**"} + tests := []struct { + path string + want bool + }{ + {"", true}, + {".env", true}, + {"config/.env.production", true}, + {"tls/server.pem", true}, + {"tls/server.KEY", true}, + {"cert.p12", true}, + {"home/.ssh/id_ed25519", true}, + {"deploy/Secrets.yaml", true}, + {"pkg/credentials/store.go", true}, + {"auth/token_cache.go", true}, + {"go.sum", true}, + {"web/package-lock.json", true}, + {"Cargo.lock", true}, + {"vendor/x/y.go", true}, + {"web/node_modules/a/b.js", true}, + {"static/app.min.js", true}, + {"api/zz_generated.deepcopy.go", true}, + {"proto/api.pb.go", true}, + {"pkg/model_gen.go", true}, + {"docs/guide.md", true}, + {"ui/__snapshots__/x.snap", true}, + {"fixtures/a/b.txt", true}, + {"src/pkg/dashboard/server.go", false}, + {"docs/sub/guide.md", false}, + {"README.md", false}, + {"pkg/vendor.go", false}, + } + for _, tc := range tests { + if got := matrixPathDenied(tc.path, append(excludes, " ")); got != tc.want { + t.Errorf("matrixPathDenied(%q) = %v, want %v", tc.path, got, tc.want) + } + } +} + +func TestMatrixAcceptLine(t *testing.T) { + tests := []struct { + name string + in string + want string + ok bool + }{ + {name: "plain", in: " return x + y ", want: "return x + y", ok: true}, + {name: "empty", in: " "}, + {name: "invalid utf8", in: "bad \xff byte"}, + {name: "control char", in: "esc \x1b[31m red"}, + {name: "tab allowed", in: "a\tb", want: "a\tb", ok: true}, + {name: "exactly 160 runes", in: strings.Repeat("ab ", 53) + "x", want: strings.Repeat("ab ", 53) + "x", ok: true}, + {name: "over 160 runes dropped not truncated", in: strings.Repeat("ab ", 54)}, + {name: "entropy run", in: "sum := " + strings.Repeat("0f", 16)}, + {name: "entropy run 31 ok", in: "id " + strings.Repeat("a", 31), want: "id " + strings.Repeat("a", 31), ok: true}, + {name: "redaction marker", in: "value here"}, + {name: "scrub changes", in: "key AKIA" + "IOSFODNN7EXAMPLE end"}, + {name: "pane scrub changes", in: "use sk-" + "abcdef12 now"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got, ok := matrixAcceptLine(tc.in) + if ok != tc.ok || got != tc.want { + t.Fatalf("matrixAcceptLine(%q) = %q, %v; want %q, %v", tc.in, got, ok, tc.want, tc.ok) + } + }) + } +} + +func TestMatrixAddedLines(t *testing.T) { + if got := matrixAddedLines(""); got != nil { + t.Fatalf("empty patch = %q", got) + } + if got := matrixAddedLines("+a\x00b"); got != nil { + t.Fatalf("binary patch = %q", got) + } + got := matrixAddedLines("+++ b/f\n@@\n context\n-removed\n+added\n+-----BEGIN X-----\n+body\n+-----END X-----\n+after") + if strings.Join(got, "|") != "added|after" { + t.Fatalf("added lines = %q", got) + } +} + +func TestMatrixLang(t *testing.T) { + for path, want := range map[string]string{"a/b.go": "go", "x.TSX": "typescript", "Makefile": "", "s.yml": "yaml"} { + if got := matrixLang(path); got != want { + t.Errorf("matrixLang(%q) = %q, want %q", path, got, want) + } + } +} + +func TestMatrixSourceDedup(t *testing.T) { + m, clk := newTestMatrixSource(t, matrixTestConfig()) + file := sentinel.File{Path: "a.go", Patch: matrixPatch("one", "two", "one")} + if got := m.ingest(matrixObs(1, "h1", file)); got != 2 { + t.Fatalf("first ingest = %d, want 2", got) + } + if got := m.ingest(matrixObs(1, "h1", file)); got != 0 { + t.Fatalf("same head re-observed added %d", got) + } + rebased := sentinel.File{Path: "a.go", Patch: matrixPatch("one", "two", "three")} + if got := m.ingest(matrixObs(1, "h2", rebased)); got != 1 { + t.Fatalf("rebased head added %d, want 1 new line", got) + } + other := sentinel.File{Path: "b.go", Patch: matrixPatch("one")} + if got := m.ingest(matrixObs(1, "h2", other)); got != 1 { + t.Fatalf("same line in another file added %d, want 1", got) + } + if got := m.ingest(matrixObs(2, "h1", file)); got != 2 { + t.Fatalf("another PR added %d, want 2", got) + } + clk.advance(matrixDedupTTL + time.Minute) + if got := m.ingest(matrixObs(3, "h", sentinel.File{Path: "c.go", Patch: matrixPatch("z")})); got != 1 { + t.Fatalf("fresh PR added %d", got) + } + if _, ok := m.prs[matrixTestRepo+"#1"]; ok { + t.Fatalf("PR set older than the TTL was not pruned") + } + if m.totalHashes != 1 { + t.Fatalf("totalHashes = %d after prune, want 1", m.totalHashes) + } + if got := m.ingest(matrixObs(1, "h2", file)); got != 2 { + t.Fatalf("after TTL expiry re-observed head added %d, want 2", got) + } +} + +func TestMatrixSourcePerHeadCap(t *testing.T) { + m, _ := newTestMatrixSource(t, matrixTestConfig()) + files := []sentinel.File{ + {Path: "a.go", Patch: matrixPatch(matrixUniqueLines("a", 150)...)}, + {Path: "b.go", Patch: matrixPatch(matrixUniqueLines("b", 150)...)}, + } + if got := m.ingest(matrixObs(1, "h1", files...)); got != matrixMaxLinesPerHead { + t.Fatalf("ingest = %d, want %d", got, matrixMaxLinesPerHead) + } + if got := m.ingest(matrixObs(1, "h1", sentinel.File{Path: "c.go", Patch: matrixPatch("more")})); got != 0 { + t.Fatalf("head over its cap still added %d", got) + } + if got := m.ingest(matrixObs(1, "h2", sentinel.File{Path: "c.go", Patch: matrixPatch("more")})); got != 1 { + t.Fatalf("new head added %d, want 1", got) + } +} + +func TestMatrixSourceRingBound(t *testing.T) { + m, clk := newTestMatrixSource(t, matrixTestConfig()) + for pr := 1; pr <= 15; pr++ { + clk.advance(matrixRateWindow) + m.ingest(matrixObs(pr, "h", sentinel.File{Path: "a.go", Patch: matrixPatch(matrixUniqueLines(fmt.Sprint(pr), matrixMaxLinesPerHead)...)})) + } + lines := matrixAllLines(m) + if len(lines) != matrixBacklogCap { + t.Fatalf("backlog = %d, want %d", len(lines), matrixBacklogCap) + } + if lines[0].Seq != 1001 || lines[len(lines)-1].Seq != 3000 || m.Latest() != 3000 { + t.Fatalf("ring order wrong: first=%d last=%d latest=%d", lines[0].Seq, lines[len(lines)-1].Seq, m.Latest()) + } + page, latest := m.Since(2990, 0) + if len(page) != 10 || page[0].Seq != 2991 || latest != 3000 { + t.Fatalf("Since(2990) = %d lines from %d, latest %d", len(page), page[0].Seq, latest) + } + page, _ = m.Since(0, 5) + if len(page) != 5 || page[0].Seq != 1001 { + t.Fatalf("Since(0, 5) = %d lines from %d", len(page), page[0].Seq) + } + page, _ = m.Since(0, -1) + if len(page) != matrixDefaultSinceMax { + t.Fatalf("Since default page = %d", len(page)) + } +} + +func TestMatrixSourceRateLimit(t *testing.T) { + m, clk := newTestMatrixSource(t, matrixTestConfig()) + total := 0 + for pr := 1; pr <= 4; pr++ { + total += m.ingest(matrixObs(pr, "h", sentinel.File{Path: "a.go", Patch: matrixPatch(matrixUniqueLines(fmt.Sprint(pr), 175)...)})) + } + if total != matrixMaxLinesPerRate || m.rateDropped != 1 { + t.Fatalf("accepted %d (dropped %d), want %d", total, m.rateDropped, matrixMaxLinesPerRate) + } + clk.advance(matrixRateWindow) + if got := m.ingest(matrixObs(5, "h", sentinel.File{Path: "a.go", Patch: matrixPatch("next window")})); got != 1 { + t.Fatalf("next window accepted %d", got) + } +} + +func TestMatrixSourceDedupBounds(t *testing.T) { + m, clk := newTestMatrixSource(t, matrixTestConfig()) + for pr := 1; pr <= matrixMaxTrackedPRs+20; pr++ { + clk.advance(time.Second) + m.ingest(matrixObs(pr, "h", sentinel.File{Path: "a.go", Patch: matrixPatch(fmt.Sprintf("pr %d", pr))})) + } + if len(m.prs) != matrixMaxTrackedPRs { + t.Fatalf("tracked PRs = %d, want %d", len(m.prs), matrixMaxTrackedPRs) + } + if _, ok := m.prs[matrixTestRepo+"#1"]; ok { + t.Fatalf("oldest PR set was not evicted") + } + + old := matrixMaxDedupHashes + matrixMaxDedupHashes = 50 + t.Cleanup(func() { matrixMaxDedupHashes = old }) + m2, clk2 := newTestMatrixSource(t, matrixTestConfig()) + for pr := 1; pr <= 4; pr++ { + clk2.advance(time.Second) + m2.ingest(matrixObs(pr, "h", sentinel.File{Path: "a.go", Patch: matrixPatch(matrixUniqueLines(fmt.Sprint(pr), 20)...)})) + } + if m2.totalHashes > matrixMaxDedupHashes || len(m2.prs) != 2 { + t.Fatalf("hash cap not held: total=%d prs=%d", m2.totalHashes, len(m2.prs)) + } + m3, _ := newTestMatrixSource(t, matrixTestConfig()) + m3.ingest(matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: matrixPatch(matrixUniqueLines("x", 60)...)})) + if m3.totalHashes != 0 || len(m3.prs) != 0 { + t.Fatalf("single over-cap PR kept: total=%d prs=%d", m3.totalHashes, len(m3.prs)) + } +} + +func TestMatrixSourceSkipsOversizedInput(t *testing.T) { + m, _ := newTestMatrixSource(t, matrixTestConfig()) + files := []sentinel.File{{Path: "big.go", Patch: matrixPatch(strings.Repeat("x ", matrixMaxPatchBytes))}} + for i := 0; i < matrixMaxFilesPerObs; i++ { + files = append(files, sentinel.File{Path: fmt.Sprintf("f%d.go", i)}) + } + files = append(files, sentinel.File{Path: "late.go", Patch: matrixPatch("beyond the file cap")}) + if got := m.ingest(matrixObs(1, "h", files...)); got != 0 { + t.Fatalf("oversized input added %d lines", got) + } +} + +func TestMatrixSourceObserveNeverBlocks(t *testing.T) { + m, _ := newTestMatrixSource(t, matrixTestConfig()) + obs := matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: matrixPatch("x")}) + done := make(chan struct{}) + go func() { + defer close(done) + for i := 0; i < matrixIngestCap+10; i++ { + m.Observe(obs) + } + }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("Observe blocked on a full channel") + } + if got := m.Dropped(); got != 10 { + t.Fatalf("Dropped() = %d, want 10", got) + } + if len(m.in) != matrixIngestCap { + t.Fatalf("channel holds %d, want %d", len(m.in), matrixIngestCap) + } +} + +func TestMatrixSourceRunNotifiesAndRecovers(t *testing.T) { + m, _ := newTestMatrixSource(t, matrixTestConfig()) + seqs := make(chan uint64, 4) + calls := 0 + m.setNotify(func(seq uint64) { + calls++ + if calls == 1 { + panic("notify boom") + } + seqs <- seq + }) + m.start() + m.start() + m.Observe(matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: matrixPatch("first")})) + m.Observe(matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: matrixPatch("second")})) + select { + case seq := <-seqs: + if seq != 2 { + t.Fatalf("notify seq = %d, want 2", seq) + } + case <-time.After(5 * time.Second): + t.Fatal("processing goroutine did not survive a panic") + } + m.close() + m.close() + select { + case <-m.done: + case <-time.After(5 * time.Second): + t.Fatal("processing goroutine did not stop") + } +} + +func TestServerMatrixObserver(t *testing.T) { + var nilSrv *Server + if nilSrv.MatrixObserver(matrixTestConfig()) != nil { + t.Fatal("nil server returned an observer") + } + s := &Server{} + if s.MatrixObserver(config.MatrixConfig{}) != nil || s.matrix != nil { + t.Fatal("disabled matrix registered an observer or built a source") + } + if s.MatrixObserver(config.MatrixConfig{Enabled: true}) != nil || s.matrix != nil { + t.Fatal("matrix without an allowlist registered an observer") + } + obs := s.MatrixObserver(matrixTestConfig()) + if obs == nil || s.matrix == nil { + t.Fatal("active matrix returned no observer") + } + src := s.matrix + t.Cleanup(src.close) + widened := matrixTestConfig() + widened.Repos = append(widened.Repos, "hivecommons/other") + if s.MatrixObserver(widened) == nil || s.matrix != src { + t.Fatal("second call rebuilt the source") + } + if !src.cfg.Load().RepoAllowed("hivecommons/other") { + t.Fatal("second call did not refresh the allowlist") + } + src.ingest(matrixObs(1, "h", sentinel.File{Path: "a.go", Patch: matrixPatch("kept")})) + if src.Latest() != 1 { + t.Fatalf("Latest() = %d, want 1", src.Latest()) + } + if s.MatrixObserver(config.MatrixConfig{}) != nil { + t.Fatal("disabling still returned an observer") + } + if lines, latest := src.Since(0, 10); len(lines) != 0 || latest != 1 || len(src.prs) != 0 { + t.Fatalf("disabling kept content: %d lines, latest %d, %d PRs", len(lines), latest, len(src.prs)) + } + if got := src.ingest(matrixObs(2, "h", sentinel.File{Path: "a.go", Patch: matrixPatch("late")})); got != 0 { + t.Fatalf("disabled source still ingested %d lines", got) + } +} + +func TestMatrixSourceExpiredHashInLivePR(t *testing.T) { + m, clk := newTestMatrixSource(t, matrixTestConfig()) + m.ingest(matrixObs(1, "h1", sentinel.File{Path: "a.go", Patch: matrixPatch("a")})) + clk.advance(matrixDedupTTL - time.Hour) + m.ingest(matrixObs(1, "h2", sentinel.File{Path: "a.go", Patch: matrixPatch("b")})) + clk.advance(2 * time.Hour) + if got := m.ingest(matrixObs(1, "h3", sentinel.File{Path: "a.go", Patch: matrixPatch("a", "b")})); got != 1 { + t.Fatalf("expired hash re-ingest added %d, want 1", got) + } + if m.totalHashes != 2 { + t.Fatalf("totalHashes = %d, want 2", m.totalHashes) + } +} diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 443387f15b..78a40aa5b9 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -116,6 +116,11 @@ type Server struct { chatGroundedOnce sync.Once chatGroundedInflight chan struct{} + // matrix is the dashboard.matrix line source (matrix_source.go); nil + // until MatrixObserver is called with an active config. + matrixMu sync.Mutex + matrix *matrixSource + port int authToken string statusMu sync.RWMutex