diff --git a/.github/scripts/integration-governance.cjs b/.github/scripts/integration-governance.cjs index 71d27716ce..03a783a4f0 100644 --- a/.github/scripts/integration-governance.cjs +++ b/.github/scripts/integration-governance.cjs @@ -9,7 +9,7 @@ const MAX_COMMITS = 1000; const MAX_AUTHOR_BATCH = 50; // This literal is inside the enrolled .github closure. A caller-supplied // preparedBy.sourceSHA must never choose a new executable verifier tree. -const TRUSTED_SCRIPTS_TREE = '689fea04d4b10295f67b9de708797343938d7e0b'; +const TRUSTED_SCRIPTS_TREE = 'f42d6489f036117030057d7ec2e254f58e21a06d'; const SOURCE = 'https://github.com/kubernetes-sigs/kubebuilder-release-tools/tree/012269a88fa4c034a0acf1ba84c26b195c0dbab4'; const sha = value => crypto.createHash('sha256').update(value).digest('hex'); const git = (args, cwd = process.cwd()) => diff --git a/.github/scripts/pr-overlap.sh b/.github/scripts/pr-overlap.sh index f5fdaa2692..e354d26979 100755 --- a/.github/scripts/pr-overlap.sh +++ b/.github/scripts/pr-overlap.sh @@ -240,10 +240,19 @@ report_body() { } } +# One-line body the existing marker comment is edited to once a PR that had +# overlaps no longer has any. A PR that never had a finding gets no comment +# at all: the full clean report goes to the step summary only (#17863). +resolved_body() { + local base=$1 + # shellcheck disable=SC2016 # backticks are literal Markdown + printf '%s\n\n### Open PR overlap check\n\n✅ Resolved: no conflicting open PRs or same-file overlaps remain against `%s`.\n' "$MARKER" "$base" +} + upsert_comment_and_label() { - local pr=$1 body=$2 has_conflicts=$3 + local pr=$1 body=$2 has_conflicts=$3 has_any=${4:-1} base=${5:-} if ((DRY_RUN)); then - printf 'DRY RUN: would update PR #%s comment and label=%s\n' "$pr" "$has_conflicts" + printf 'DRY RUN: would update PR #%s comment and label=%s findings=%s\n' "$pr" "$has_conflicts" "$has_any" printf '%s\n' "$body" return 0 fi @@ -251,7 +260,13 @@ upsert_comment_and_label() { local comment_id comment_id=$(api "repos/${REPO}/issues/${pr}/comments?per_page=100" --paginate \ --jq ".[] | select(.body | contains(\"${MARKER}\")) | .id" | head -n1) - if [[ -n "$comment_id" ]]; then + if [[ "$has_any" != '1' ]]; then + if [[ -n "$comment_id" ]]; then + api -X PATCH "repos/${REPO}/issues/comments/${comment_id}" -f body="$(resolved_body "$base")" >/dev/null + else + printf 'PR #%s: no overlap findings and no earlier comment; not posting\n' "$pr" + fi + elif [[ -n "$comment_id" ]]; then api -X PATCH "repos/${REPO}/issues/comments/${comment_id}" -f body="$body" >/dev/null else api -X POST "repos/${REPO}/issues/${pr}/comments" -f body="$body" >/dev/null @@ -276,12 +291,12 @@ write_summary() { run_for_one_pr() { local pr=$1 base=$2 writable=$3 truncated=${4:-0} - local rows has_conflicts _has_any base_conflict body - make_report_for_pr "$pr" "$base" rows has_conflicts _has_any base_conflict + local rows has_conflicts has_any base_conflict body + make_report_for_pr "$pr" "$base" rows has_conflicts has_any base_conflict body=$(report_body "$pr" "$base" "$rows" "$truncated" "$base_conflict") write_summary "$body" if [[ "$writable" == '1' ]]; then - upsert_comment_and_label "$pr" "$body" "$has_conflicts" + upsert_comment_and_label "$pr" "$body" "$has_conflicts" "$has_any" "$base" fi } diff --git a/.github/scripts/test-pr-overlap.sh b/.github/scripts/test-pr-overlap.sh index f3581977cc..3b553a279c 100755 --- a/.github/scripts/test-pr-overlap.sh +++ b/.github/scripts/test-pr-overlap.sh @@ -109,7 +109,7 @@ case "$*" in *'.head.repo.full_name'*) printf '%s\n' "$REPO_SLUG" ;; *'.head.sha'*) printf '%s\n' "${FALLBACK_SHA:-}" ;; *'.base.ref'*) printf 'v6\n' ;; - *'/comments?'*) : ;; + *'/comments?'*) if [[ -n "${EXISTING_COMMENT_ID:-}" ]]; then printf '%s\n' "$EXISTING_COMMENT_ID"; fi ;; *) echo "unexpected gh api call: $*" >&2; exit 2 ;; esac GH @@ -134,6 +134,8 @@ run_pr() { # pr -> sets OUT and WRITES LABEL_ADD="-X POST repos/${REPO_SLUG}/issues/%s/labels -f labels[]=conflicts-with-open-pr" LABEL_DEL="-X DELETE repos/${REPO_SLUG}/issues/%s/labels/conflicts-with-open-pr" +COMMENT_POST="-X POST repos/${REPO_SLUG}/issues/%s/comments -f body=" +COMMENT_PATCH="-X PATCH repos/${REPO_SLUG}/issues/comments/%s -f body=" echo 'PR 2: real sibling conflict with PR 3, stale PR 1 is not blamed' run_pr 2 @@ -144,6 +146,16 @@ assert_not_contains "$OUT" '| #1 |' 'does not blame PR 1 for its conflict with t assert_not_contains "$OUT" '| #4 |' 'does not link the unrelated stale PR 4' # shellcheck disable=SC2059 assert_contains "$WRITES" "$(printf -- "$LABEL_ADD" 2)" 'labels PR 2 conflicts-with-open-pr' +# shellcheck disable=SC2059 +assert_contains "$WRITES" "$(printf -- "$COMMENT_POST" 2)" 'posts the overlap comment when there is a finding' + +echo 'PR 2: an existing marker comment is edited, not duplicated' +EXISTING_COMMENT_ID=4242 run_pr 2 +assert_exit0 +# shellcheck disable=SC2059 +assert_contains "$WRITES" "$(printf -- "$COMMENT_PATCH" 4242)" 'edits the existing marker comment' +# shellcheck disable=SC2059 +assert_not_contains "$WRITES" "$(printf -- "$COMMENT_POST" 2)" 'does not post a second comment' echo 'PR 5: add/add of the same new path with different content' run_pr 5 @@ -167,9 +179,20 @@ assert_contains "$WRITES" "$(printf -- "$LABEL_DEL" 1)" 'removes the sibling-con echo 'PR 4: stale but clean, no overlap' run_pr 4 assert_exit0 -assert_contains "$OUT" 'No conflicting open PRs or same-file overlaps were found.' 'reports no overlap' +assert_contains "$OUT" 'No conflicting open PRs or same-file overlaps were found.' 'reports no overlap in the summary' # shellcheck disable=SC2059 assert_contains "$WRITES" "$(printf -- "$LABEL_DEL" 4)" 'removes the sibling-conflict label' +assert_not_contains "$WRITES" '/comments' 'posts no comment when there is nothing to report (#17863)' +assert_contains "$OUT" 'no overlap findings and no earlier comment; not posting' 'logs the skipped post' + +echo 'PR 4: a clean run edits an earlier finding comment to a one-line resolved note' +EXISTING_COMMENT_ID=777 run_pr 4 +assert_exit0 +# shellcheck disable=SC2059 +assert_contains "$WRITES" "$(printf -- "$COMMENT_PATCH" 777)" 'edits the existing comment, keeping the marker' +assert_contains "$WRITES" 'Resolved: no conflicting open PRs or same-file overlaps remain against `v6`' 'collapses to the resolved line' +# shellcheck disable=SC2059 +assert_not_contains "$WRITES" "$(printf -- "$COMMENT_POST" 4)" 'does not post a new comment' echo 'PR 3: missing pull ref falls back to the API head and replaces a stale local ref' export FALLBACK_SHA diff --git a/.github/workflows/feedback.yml b/.github/workflows/feedback.yml index 0d4f5de803..14872a8e7d 100644 --- a/.github/workflows/feedback.yml +++ b/.github/workflows/feedback.yml @@ -14,5 +14,35 @@ permissions: pull-requests: write jobs: + # The reusable workflow thanks every merged PR unconditionally; the gate + # keeps the comment for a person's first merged PR only (#17863). Rules and + # knobs: src/scripts/feedback-gate.sh. + gate: + if: github.event.pull_request.merged == true + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + contents: read + pull-requests: read + outputs: + post: ${{ steps.gate.outputs.post }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Self-test feedback gate + run: bash src/scripts/test-feedback-gate.sh + - id: gate + name: Decide whether to thank the author + env: + GH_TOKEN: ${{ github.token }} + FEEDBACK_SKIP_BOTS: '1' + FEEDBACK_SKIP_BRANCHES: 'scanner/* hive/*' + FEEDBACK_SKIP_HIVE_ORIGIN: '1' + FEEDBACK_FIRST_TIME_ONLY: '1' + run: bash src/scripts/feedback-gate.sh + feedback: + needs: gate + if: needs.gate.outputs.post == 'true' uses: hivecommons/infra/.github/workflows/reusable-feedback.yml@f1c0b61675386f54e63c04cf07b3aa933cc963eb # hivecommons/infra main as of 2026-09-02 diff --git a/CHANGELOG.md b/CHANGELOG.md index 8c2c2f6d04..712c4c86a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -124,6 +124,58 @@ Hive did not historically maintain a complete changelog. This file starts a prag - Contribute "Your army" card: a remote agent's "PRs awaiting rework" now lists only its open, not-yet-dispatched PRs — merged, closed and hive-adopted PRs are dropped (and a queued fix for a PR that is gone leaves the fix queue instead of reaching the relay); the line says why a fix is not dispatched (offline, no fix-pr capability, scope, orphaned dispatch, backoff, focus window, waiting for `ready`) instead of "not yet dispatched"; a fix dispatched to a socket that dropped is released at the relay's next `ready` and a completion reported on a reconnected socket settles it; links collapse to three inline plus a "show N more" disclosure. ## 2026-11-12 (v6.21.0) +## 2026-11-12 (v5.237.2) + +### Fixed + +- A kick that outwaits an agent's still-running turn no longer logs a WARN pane dump, so busy agents stop filing `agent_pane_timeout` self-reports; panes that never reach a prompt for other reasons still dump at WARN (#17993). + +## 2026-11-12 (v5.237.1) + +### Fixed + +- Canceled stage-label check digest reads remain in local logs without generating repeated unclassified fleet reports; other digest failures remain reportable (#17944). +- Fleet report: a GitHub App that lacks the `statuses: write` permission (the `prow/lgtm` status 403) is now reported under its own `github_app_statuses_permission_missing` class with a remediation hint instead of the catch-all `unclassified_error_repeated` (#17973). + +## 2026-11-12 (v5.237.0) + +### Added + +- The crux inference translator now records an OTel `crux.proxy.inference.requests` counter and `crux.proxy.inference.duration` histogram with bounded `status_class` and `outcome` attributes, plus one server span per request, following the `crux serve` HTTP metrics pattern; nothing is exported unless `OTEL_*` is configured (hivecommons/crux#153). + +## 2026-11-12 (v5.236.5) + +### Fixed + +- Avoid false repeated-error self-reports for structured context cancellations, including wrapped cancellations, while retaining local warnings (#17935, #17944, #17949). + +## 2026-11-12 (v5.236.4) + +### Security + +- The per-agent pluk session log under `/var/run/pluk/logs` is now prepared without following symlinks: the open uses `O_NOFOLLOW`, the descriptor is checked to be a regular file, and the owner and shared mode are applied through that descriptor rather than by path. The logs directory is group-writable by every agent uid, so a path-based chmod/chown let one agent widen any `dev`-owned 0600 file (the MITM CA key, the installation-token cache) to the shared group by linking its own session log at it; a planted symlink or FIFO is now replaced by a regular log instead (#17966). + +## 2026-11-12 (v5.236.3) + +### Fixed + +- Avoid repeated fleet reports for canceled in-place adoption access reads while retaining other access errors. +- Per-agent GitHub App token mints that fail at the transport layer (dial timeout, connection refused, context deadline) now raise a `forge_unreachable` condition instead of `github_app_token_mint_failed`, so an egress outage no longer tells operators to re-check their App permissions. + +## 2026-11-12 (v5.236.2) + +### Fixed + +- Classify repeated missing Dependabot alerts permission warnings as an operator-action self-report with GitHub App remediation, rather than an unclassified code defect. +- Avoid reporting unsupported signed-commit rewrites as API errors and explain how to sign the unchanged branch safely. + +## 2026-11-12 (v5.236.1) + +### Changed + +- docs: ADR-0026 Substrate actors run credential-free; hive brokers pushes post-kick (#17934) + +## 2026-11-12 (v5.236.0) ### Added @@ -161,18 +213,249 @@ Hive did not historically maintain a complete changelog. This file starts a prag - Hub marketplace seeds curated workflow models from `pkg/hub/marketplace/curated/*.yaml` on startup and every refresh, ships the first one — `cncf-techdocs-assessment` (CNCF TechDocs AI-assisted assessment) — and adds `cncf/techdocs` to the ingest corpus via the new operator-added `extra.json` list (hivecommons/hive#17620). ## 2026-11-12 (v6.19.2) +### Changed + +- Claude auth: while `HIVE_CLAUDE_OAUTH_TOKEN` (or its secret file) is set, Claude agents run on the operator's long-lived token and the shared `~/.claude/.credentials.json` is not what they authenticate with, so the per-agent credential re-bridge (launch, watchdog poll and post-sign-in) and the durable-credential watchdog's Claude probe are suspended: no credential files are moved, no `claude_token_missing` audit or "durable credential unusable" warning is raised for Claude. The suspension is logged once at startup; Copilot watching is unchanged, and with the token unset everything behaves exactly as before. (#17760) + +### Fixed + +- Docs: `releases.md` no longer claims candidate hives bypass the daily/weekly auto-upgrade cadence by default; since #11816 they follow their stored mode, and the page now explains how to switch a hive to `instant`. +- Hosted hives accept GitHub webhooks relayed by the hub: the hub signs each relay with a per-hive relay key delivered over the heartbeat (never the App webhook secret), and the spoke verifies it with timestamp and delivery-ID replay protection, turning on event-driven review dispatch for hosted hives. +- Inbox: the panel no longer sits on "Loading inbox…" forever on a hive with no voice connector configured — the Inbox service now always boots, `GET /api/inbox` returns `200` with an empty list, and a non-OK response renders a visible error instead of the loading placeholder. + +## 2026-11-12 (v5.235.1) + +### Fixed + +- `hive-baseline-check.sh` no longer always exits 2 under the gh wrapper ([#17835](https://github.com/hivecommons/hive/issues/17835)). Its sibling-PR lookup (`gh pr list --state open --json …statusCheckRollup`) was refused with `enumeration_disabled`, so the kick's mandatory baseline triage never produced a verdict. The helper now marks that one call with `HIVE_BASELINE_CHECK=1` and the wrapper lets only that narrow projection through (no `--search`; every other `gh pr list` stays blocked), and a wrapper policy block is now reported distinctly from a GitHub API failure. + +## 2026-11-12 (v5.235.0) + +### Added + +- Wispr Flow facts include recording timestamps and a transcript section for voice Inbox ingestion. + +## 2026-11-12 (v5.234.1) + +### Fixed + +- `check-action-pins.sh` no longer reports a SHA-pinned action as `MISSING-SHA` when the GitHub API merely failed to answer. Only a definite `HTTP 404`/`HTTP 422` is a missing pin; a rate limit, a 5xx or a dropped connection is retried three times and then reported as `UNRESOLVED @ … re-run, this is not a missing pin`, so a transient API error on a docs-only commit can no longer read as a fabricated pin that needs bumping (it sent hivecommons/crux `main` red on a README change whose `actions/setup-go` pin was tagged `v7.0.0` upstream all along). + +## 2026-11-12 (v5.234.0) + +### Added + +- The Inbox now reads Plaud recordings directly from the connector (keyed by the Plaud file id) while it is signed in, merging them with the synced facts and refreshing the listing in the background so dashboard polls never wait on the Plaud process. + +## 2026-11-12 (v5.233.0) + +### Added + +- Settings → Inbox section to edit the default RFC repository, dibs URL and auto-routing rules from the dashboard Settings dialog. + +## 2026-11-12 (v5.232.0) + +### Added + +- Knowledge → Integrations: the Plaud card now links to the Inbox and voice sources quickstart (`inbox-voice-sources.md`) ([#17882](https://github.com/hivecommons/hive/issues/17882)). + +## 2026-11-12 (v5.231.1) + +### Changed + +- PR comment noise: the three per-PR workflow comments that survived the hive-App reducers now post only on a finding ([#17863](https://github.com/hivecommons/hive/issues/17863)). `.github/scripts/pr-overlap.sh` creates its `` comment only when a sibling PR conflicts or shares a file and, when a later push clears that, edits the existing comment to a one-line "Resolved" note instead of re-posting "no overlaps found" on every PR; `src/scripts/mutation-report.sh post` does the same for `` on a `pass` verdict (the full report, clean case included, still goes to the step summary); the "Thank you for your contribution!" merge comment (`feedback.yml`) is gated by the new `src/scripts/feedback-gate.sh` to a person's first merged PR — bot/App authors, `scanner/*` and `hive/*` branches, hive-origin bodies and returning contributors are skipped, with `FEEDBACK_*` workflow `env` knobs. A dashboard approval on a Prow-managed PR the hive account opened now relays `/approve` only, since a self-`/lgtm` is rebuked by Prow every time (`PullRequestApprovalEvent.LGTMSkipped`). Expected effect: roughly half of the comments on a typical merged PR. + +### Fixed + +- Config saves now write `/etc/hive/hive.yaml` and `/data/hive.yaml.runtime` atomically (temp file + fsync + rename onto the symlink target), the entrypoint refreshes the legacy `/data/hive.yaml.bak` fallback from a validated runtime config on every clean boot, and falling back to it logs a loud WARN — a liveness kill mid-save no longer boots a months-old ACMM level and roster (#17862). + +## 2026-11-12 (v5.231.0) + +### Added + +- **Inbox: voice recorders as first-class input sources.** A new 📥 Inbox section (sidebar → Intelligence) lists recordings and dictations synced by voice knowledge connectors (Plaud today, Wispr Flow next) keyed by `(source, item_id)`, and lets the owner file each one as a spektacular RFC with child issues on a chosen repo (`spektacular voice --publish`, run inside the hive or handed to a spoke agent), send it to dibs as a draft idea, or keep it as a Knowledge fact only — single or bulk, with per-source auto-routing rules (`inbox.rules`) applied after every sync. The Plaud card in Knowledge → Integrations now only shows connection state plus a paste-to-finish sign-in for hosted hives (`POST /api/knowledge/connectors/{name}/auth/complete`), since Plaud's OAuth client only redirects to `localhost:8199`. New routes `GET /api/inbox`, `POST /api/inbox/route`, `POST /api/inbox/dismiss`, `GET|POST /api/inbox/settings`; docs in `inbox-voice-sources.md`. + +## 2026-11-12 (v5.230.0) + +### Added + +- Settings → Workflows → Library gains an owner-only **Derive stages from bound models** switch (`workflow.enabled`, [#17620](https://github.com/hivecommons/hive/issues/17620)) backed by the new `PUT /api/workflows/enabled {enabled}`, so a hosted hive with no `hive.yaml` access can turn model-based stage derivation on or off from the dashboard. Off: a binding only changes the Activity Monitor's columns; on: a bound repository's stages are derived by its model's rules and the model's labels are provisioned in the repository immediately. Persists `hive.yaml`, takes effect on the next derivation pass without a restart, audited as `workflow_enabled` / `workflow_disabled`; the Repos binding row now says *columns only — turn on derivation in Workflows* when a repository is bound while the flag is off. + +### Fixed + +- Classify repeated plain HTTP proxy forwarding failures separately, with diagnostics for the requested host and network path. + +## 2026-11-12 (v5.229.0) + +### Added + +- Knowledge: new `wisprflow` connector syncs Wispr Flow dictation history (text, time, source app) into the vault from the desktop app's local `flow.sqlite` or an export folder — read-only, no sign-in, one fact per dictation or per day, with a Knowledge → Integrations card; dictating straight into **+ New Fact** needs no connector at all. + +### Fixed + +- Agents whose pluk session log under `/var/run/pluk/logs` was left behind by another uid (for example the supervisor's `hive-supervisor.jsonl`) no longer fail log setup with `permission denied`; the stale file is replaced with a peer-readable one so `hive-panes` and the dashboard can read it ([#17871](https://github.com/hivecommons/hive/issues/17871)). + +## 2026-11-12 (v5.228.6) + +### Fixed + +- Detect missing GitHub App repository coverage at boot, during self-heal, and when connecting a repository, with a settings link to fix the installation selection (#17696). + +## 2026-11-12 (v5.228.5) + +### Fixed + +- Contribute "Your army" card: a remote agent's "PRs awaiting rework" now lists only its open, not-yet-dispatched PRs — merged, closed and hive-adopted PRs are dropped (and a queued fix for a PR that is gone leaves the fix queue instead of reaching the relay); the line says why a fix is not dispatched (offline, no fix-pr capability, scope, orphaned dispatch, backoff, focus window, waiting for `ready`) instead of "not yet dispatched"; a fix dispatched to a socket that dropped is released at the relay's next `ready` and a completion reported on a reconnected socket settles it; links collapse to three inline plus a "show N more" disclosure. + +## 2026-11-12 (v5.228.4) + +### Security + +- The Plaud knowledge connector's stdio MCP server (`npx -y @plaud-ai/mcp@latest` by default) now starts with an allowlisted environment (`PATH`, `HOME`, locale, proxy/CA variables, `NODE_*`, `npm_config_*`, `PLAUD_*`) instead of the full hive process environment, so hub-injected credentials such as `DASHBOARD_AUTH_TOKEN`, `HIVE_TERMINAL_KEY` and `HIVE_UPSTREAM_PROXY_URL` no longer reach third-party code fetched at run time ([#17843](https://github.com/hivecommons/hive/issues/17843)). + +## 2026-11-12 (v5.228.3) + +### Security + +- Bound the hub's GitHub bearer-token validation cache and reclaim expired entries before admitting new tokens, preventing unbounded cache growth from unauthenticated requests. + +## 2026-11-12 (v5.228.2) + +### Fixed + +- crux serve: requests refused with a 401 by authentication are now counted in the new `crux.serve.auth.rejections` metric, labelled only by a fixed `reason` (`missing_or_invalid_token`, `project_forbidden` or `no_token_configured`); they never reached `crux.serve.requests` before. +- The hub now remembers a rejected or rate-limited `Bearer` token (60 s, or until its `X-RateLimit-Reset` capped at 15 min; [#17836](https://github.com/hivecommons/hive/issues/17836)) instead of re-probing `GET /user` with that token on every request, which charged the token owner's already-exhausted core quota and prolonged the outage; the probes are visible in `/api/gh-rate-limits` as caller `agent:hub-bearer-validate`, and hub release-notes contents reads now go through the shared ETag/accounting transport. + +## 2026-11-12 (v5.228.1) + +### Fixed + +- hub: a reconcile sweep now converges the template startupProbe and the current init-container image repository onto pre-existing hosted hive Deployments, so a slow boot no longer becomes a permanent liveness crash-loop (#17827) +- Substrate `snapshot_on_idle` no longer accumulates suspended actors: the previous idle snapshot of an agent is deleted when the next one is taken (actors from before a hive restart are not reclaimed). + +## 2026-11-12 (v5.228.0) + +### Added + +- Knowledge: new `plaud` connector syncs Plaud Note voice recordings (Plaud summary + transcript) into the vault through the free official Plaud MCP server, with Connect/Disconnect sign-in on the Knowledge → Integrations card and no cost beyond the Plaud plan. + +### Fixed + +- The hive binary no longer delays the dashboard bind on the startup advisory-issue ensure: it is now capped at 15s, so a slow or unreachable GitHub Enterprise (~8s per App token mint) can no longer push `:3002` past the kubelet liveness budget and crash-loop the pod ([#17827](https://github.com/hivecommons/hive/issues/17827)). + +## 2026-11-12 (v5.227.1) + +### Fixed + +- A hive whose ID the hub refuses to auto-create (heartbeat 403 "hosted/saas hive IDs can only be created via provisioning") now backs off to one probe every 10 minutes and reports a dedicated `hub_heartbeat_rejected_unprovisioned` fleet-report class, instead of warning every tick and filing `unclassified_error_repeated` ([#17815](https://github.com/hivecommons/hive/issues/17815)). +- Persisting hive state and reach state no longer fails with "rename ... no such file or directory" when two writers race: each write uses a unique temp file before the atomic rename (#17726, #17728). + +## 2026-11-12 (v5.227.0) + +### Added + +- The dashboard's read-only terminal pane now covers `sandbox.runtime: substrate` agents: it shows the kick's Substrate actor (name, state, worker SPIFFE id, suspend/resume transitions) and streams the kick's output live from the actor bridge's `/run`. + +### Fixed + +- Hub "idle with work queued" no longer flags agents the governor will never kick: agents with no (or a paused) cadence in the current governor mode are skipped, and spokes now report the slot-to-slot gap for times/cron cadences so a `09:00,12:00,15:00 weekdays` agent is not alerted every evening and all weekend. +- Spoke self-upgrade no longer fails with `reading startup probe before upgrade … EOF` on nodes without the `xt_owner` netfilter module (IBM Cloud IKS/ROKS): the hive's in-cluster Kubernetes API clients now SO_MARK their sockets like the proxy's own upstream dials, so the forced-egress REDIRECT exempts them instead of routing them into the MITM proxy (shared `crux/egressmark`). + +### Security + +- Hub `POST /api/marketplace/refresh?repo=` from a spoke (heartbeat bearer) now refuses with 403 any repository the hub does not already track (the CNCF corpus plus `HIVE_HUB_MARKETPLACE_EXTRA_REPOS`); previously any registered spoke could make the hub fetch an arbitrary GitHub repository with the hub's own GitHub identity and publish it to the shared catalogue under the hub's name. Hub admins can still refresh any repository. +- The Substrate sandbox runtime no longer follows symlinks when unpacking a workspace pulled from an actor, so a sandboxed kick cannot write outside the local workspace on the hive host. + +## 2026-11-12 (v5.226.0) + +### Added + +- Dashboard: agent cards now show the Substrate (and OpenShell) sandbox preflight warnings for the agent's runtime, exposed as `sandbox_warnings` in `/api/status` from the boot-time sandbox posture check (hivecommons/hive#17779). + +### Fixed + +- The hub heartbeat `app_keys_held` collector no longer re-reads and re-parses every per-app GitHub App key file on each beat (#17797): fingerprints are cached by file size and mtime, so a slow `/data` volume stops pushing it past the 10s collect budget and flooding the "hub heartbeat collect budget exceeded" WARN. +- Repeated `review swarm kick not delivered` warnings for the same PR head and cause are now logged once (then at debug) instead of every cycle. + +## 2026-11-12 (v5.225.0) + +### Added + +- Hub `clusters.json` accepts per-cluster `host_aliases` (`[{ip, hostnames[]}]`, validated at load) rendered as Pod `hostAliases` on every hosted-hive Deployment, plus a rate-limited reconcile sweep that converges existing hives — for GitHub Enterprise hosts reachable only through a TLS reverse proxy on a fixed IP, where `upstream_proxy` cannot help (#17801). +- Dashboard Settings: surface the `agent_sandbox.substrate.*` and per-agent `agents..sandbox.substrate.*` leaves (endpoint, atespace, token, TLS, router, bridge, resources, worker selector, egress hosts, policy profile) in the Settings dialog. + +### Fixed + +- Repeated agent-lifecycle log lines (failed kick send, bare-shell crash, missing tmux session, kill-session failure, watchdog observe-mode restart) now file under the `agent_lifecycle_repeated` class instead of `unclassified_error_repeated`. +- Automerge no longer waits forever when GitHub leaves a head's workflow runs orphaned in `queued` with zero jobs: once every same-repo run is older than the never-started grace (30m) without a job, the sweep treats the head as never started so the existing rebase-nudge / bot-takeover recovery and terminal hold apply instead of the bare `required-check-missing` reason (#17775). +- Claude auth on per-UID hives: a Claude Code token refresh writes a temp file and renames it over `~/.claude/.credentials.json`, which replaced the per-agent symlink to the shared login with a private 0600 file; the refreshed token (and the rotated refresh grant) never reached the shared credential and the fleet logged out about every eight hours. The hive now detects the replaced link at agent launch, every 5s from the credential watchdog (`HIVE_CLAUDE_CREDENTIAL_BRIDGE_INTERVAL`) and after a dashboard sign-in, reads the agent's private file as its owner through `su-exec` when the server cannot open it, publishes the newer usable token into the shared login (the previous shared file is kept as `.credentials.json.stale-`), moves the private copy aside and restores the link. Per-agent `~/.claude` memory stays isolated; only the login is shared, with no operator switch. Audited as `claude_credential_rebridged`. Also hardens the shared-home permission guard and credential promotion against non-regular files: shared logins and CLI configs are written through a randomly named temporary with fd-based mode/owner, the root guard only repairs regular files in regular directories, and the shared dot-dirs get the sticky bit. +- DCO: a PR author's own signed commit is no longer filed as the hive's to re-sign after a hive rebase makes the App the committer (a sign-off with the commit author's own email on a commit GitHub links to the PR author now counts, matching the squash attribution gate), and the dashboard no longer shows a `Needs human · decision` pill or "waiting on 🐝 hive reviewer" on a row parked on `hive-signoff-attribution` or `dco-missing` — the waiting-on pill names the author instead. +- The hub activity digest now reads the last stage-tracker body immediately when the cache is stale and rebuilds in the background, so heartbeats stop dropping the activity block when a rebuild takes longer than 5 s. +- Log the issue-claims GitHub side-effect retry at INFO instead of WARN; only exhausted retries warn, so expected transient timeouts no longer trip the fleet-report repeated-error classifier (#17782). + +### Security + +- Workflow marketplace: a bundle's hash-covered `spec.meta.provenance` can no longer launder a vouched tier past the hub's admin-only policy. The hub refuses (403) a non-admin publish whose model claims `curated` or `built-in`, and a spoke importing a bundle (from the hub or by direct upload) stores a vouched provenance only when the envelope tier asserts the same tier; otherwise it falls back to the envelope tier, so the library badge "Reviewed and vouched for by a human" reflects a hub admin's vouching, not the publisher's claim. + +## 2026-11-12 (v5.224.0) + +### Added + +- Benchmark records exported by `GET /api/benchmarks/export` (and pushed to hivecommons/benchmarks) now carry an estimated cost (#17790, #17694): `cost.basis` is `tokens_x_pricing` with `cost.usd_est`, token share, `launches` and `price_table_date` when the launch-cost collector attributed a kick to the PR (the swarm's PR ref, or the governor's issue ref of an issue the PR closed), else `unknown`. Figures are list-price estimates from the spoke's own launch attribution, never billed amounts; no per-session usage leaves the spoke. +- Hub marketplace seeds curated workflow models from `pkg/hub/marketplace/curated/*.yaml` on startup and every refresh, ships the first one — `cncf-techdocs-assessment` (CNCF TechDocs AI-assisted assessment) — and adds `cncf/techdocs` to the ingest corpus via the new operator-added `extra.json` list (hivecommons/hive#17620). ### Fixed - Marketplace ingest streams each repository tarball and keeps only pipeline files (1 MiB per file, 32 MiB total) instead of extracting the whole source tree under a 256 MiB cap, so large monorepos (kubernetes/kubernetes, kserve, meshery, …) no longer fail with `archive exceeds size cap`; `GET /api/marketplace/status` now lists failing repos with their errors without `?verbose=1`; four renamed corpus repositories (composefs, OpenMetrics, ratify, kuberay) now point at their current GitHub names. ## 2026-11-11 (v6.19.1) +### Security + +- The upstream forward-proxy loader no longer echoes the configured proxy URL's `user:password` userinfo in the error it returns for a malformed URL (unsupported scheme such as `socks5://`, missing host, or an unparseable value). Boot logs that error to the hive JSON log before falling back to direct egress, so a typo in `network.upstream_proxy.url` / `HIVE_UPSTREAM_PROXY_URL` used to write the corporate proxy's Basic credential to the log stream on every start; the password is now masked as `xxxxx` (#17730). + +## 2026-11-11 (v5.223.0) + +### Added + +- Extend the `sandbox.runtime: substrate` preflight into a cluster prover (#17698): when hive runs in-cluster it now also reports, read-only and cached per control plane, missing Substrate CRDs or operator Deployments/atelet, WorkerPools with no ready replica, a cluster without gVisor (no `gvisor` SandboxConfig or `runsc` RuntimeClass) or without KVM for a microVM pool, node kernels below gVisor's floor, the `PodCertificateRequest`/`ClusterTrustBundle` APIs not being served, and an atenet-router that is unreachable or rejects hive's client certificate; each is one actionable warning. `deploy/k8s/sandbox-substrate-preflight-rbac.yaml` grants the read access it needs Two fixes found by the first real kick on a Substrate v1.0.0 cluster (#17697) ride along: the per-actor egress policy now carries the actor's atespace (the Control API rejects it otherwise) and the in-actor bridge empties `/workspace` in place instead of unlinking the durableDir mount point. ### Fixed - Marketplace: the CNCF catalogue now actually fills — tarball extraction recognises git's `pax_global_header`, so pipeline files are read for every repository (previously only tektoncd/pipeline survived); seeding resumes across hub restarts by ingesting only repositories that are due (no record, stale, old format, or an hour-old error); the last refresh summary is persisted so `/api/marketplace/status` survives a restart and reports `pendingRepos`/`lastRefreshErrors`, with `?verbose=1` listing failing repos and their errors; `POST /api/marketplace/refresh` accepts a spoke's heartbeat credential (rate-limited, `429` + `Retry-After`) so a hive owner's "Refresh catalogue" works, and the crawl backs off on `403`/`429` `Retry-After` and logs progress every 25 repositories. ## 2026-11-11 (v6.19.0) +- OpenShell runtime: sandbox names now fit the gateway's 19-character limit (`hk--`), the shared claim mounts as `hive-workspace` instead of the reserved `workspace`, and the deploy doc covers the Agent Sandbox prerequisite, `allowed_pod_labels`/annotations, sandbox uid, workspace storage class and the measured #17643 latency (#17643) +- The "contributor-lane ok-to-test not posted again" warning is logged once per PR rejection, not again after a restart once the PR already records it, so it no longer trips the repeated-error selfreport (#17738). +- The status builder publishes a (possibly stale) snapshot after a bounded number of consecutive stale drops instead of livelocking and warning on every drop during mutation bursts (#17727). + +## 2026-11-11 (v5.222.1) + +### Fixed + +- The effect ledger no longer leaves an `issue_create` op stuck in Unknown forever after an ambiguous GitHub response: once a refused retry finds it unresolved past the kind's TTL (1h), the boundary expires it to NotApplied, clears the `effect_ledger_op_stuck` self-report and lets the caller's next attempt (which re-runs its own duplicate search first) create, without ever retrying the ambiguous write in the refused call (#17693). +- Good-first-issue curation skips archived read-only repositories after GitHub refuses a write, instead of repeatedly reporting failures. +- Fixed a startup panic in the stage engine when hive ran with no loaded config, which made the auto-merge sweep test flaky (#17736). +- A read-only integration lane no longer retries removing the hive:unauthorized label every cycle and warning each time; the removal is skipped and left to a write-capable lane with a single debug line (#17720). + +## 2026-11-11 (v5.222.0) + +### Added + +- Add Agent Substrate (CNCF Sandbox) as a fourth, opt-in `sandbox.runtime: substrate` (ADR-0026 phase 2). Actors are created through Substrate's gRPC Control API with a hive-derived `atenet-egress` policy, a digest-pinned image, and workspace transfer through an in-actor bridge over the atenet-router mTLS tunnel; `sandbox.substrate.snapshot_on_idle` suspends instead of deleting the actor. Credential brokering is unchanged (agents still use the hive MITM proxy). No new Go modules: a trimmed wire-compatible proto subset is generated in-repo. + +### Changed + +- Document Agent Substrate as the opt-in fourth `sandbox.runtime` (ADR-0026 phase 2): ADR amendment with the contract mapping, deviations and when to choose it over OpenShell, a `sandbox-isolation.md` section, `deploy/substrate.md` install and verification notes with the hive cluster survey, and the integrations table entry. + +## 2026-11-11 (v5.221.1) + +### Fixed + +- Copilot agents no longer crash-loop on hosts whose kernel refuses the Copilot CLI binary with ENOEXEC (seen fleet-wide on OKE after the UEK8 / Linux 6.12 node upgrade: the Bun single-file ET_EXEC the `@github/copilot` platform package ships is rejected by the 6.12 ELF loader, the npm shim's `sh` fell back to "running" the ELF as a script, and every copilot agent sat in "copilot hung with no output" under the restart-storm breaker). The hive image now wraps the platform binary so it starts through glibc's `ld.so` (`bin/install-copilot-exec-shim.sh`, arch-aware, idempotent, `HIVE_COPILOT_DIRECT_EXEC=1` to bypass), the copilot hang diagnostic recognises `Exec format error` / the `ld-linux-…: not found` fallback signature and fails the agent loudly with the remedy instead of relaunching, and the in-process runtime execs every configured agent CLI once at boot and logs ERROR with the same remedy when the kernel refuses one (#17729, #17642). +- The model-discovery self-report no longer carries raw binary when the Copilot CLI helper is refused with ENOEXEC; the error now names the cause (#17735). +- The "agent git config declares a user identity outside the system gitconfig" warning is logged once per agent and config, not every cycle, so it no longer trips the repeated-error selfreport (#17703). + +## 2026-11-11 (v5.221.0) ### Added diff --git a/bin/gh-wrapper.sh b/bin/gh-wrapper.sh index 2f862e3639..f1b1805f5e 100755 --- a/bin/gh-wrapper.sh +++ b/bin/gh-wrapper.sh @@ -592,6 +592,61 @@ _pr_list_merged_only() { [[ "$(_lower "$state")" = "merged" ]] } +# The one `gh pr list` that bin/hive-baseline-check.sh makes for its sibling-PR +# lookup (#17835): the kick mandates that helper before any check is retried, +# and names that lookup as the sanctioned exception to the enumeration ban. The +# helper marks its call with HIVE_BASELINE_CHECK=1. That marker is a caller-set +# hint, not a trust boundary — any agent can set it, exactly as with +# HIVE_CONTRIBUTOR_MODE above — so what makes the exception safe is that only +# the helper's exact projection is let through, checked as an allowlist that a +# forgotten flag cannot widen: the sole flags accepted are --repo/-R, --state/-s +# (must resolve to open), --limit/-L (at most 100, what the helper sends) and +# --json whose every field is one of number, headRefOid, isCrossRepository, +# statusCheckRollup. Any other flag or positional (--label, --search, --author, +# --base, -q, ...) refuses, since each would turn it into a general work queue. +_pr_list_baseline_check() { + [[ "${HIVE_BASELINE_CHECK:-}" = "1" ]] || return 1 + [[ "$subcmd" = "pr" && "$action" = "list" ]] || return 1 + local i a state="" json="" limit="" seen_sub=0 seen_act=0 + for ((i=0; i<${#args[@]}; i++)); do + a="${args[$i]}" + case "$a" in + --repo=*|-R=*|-R?*) ;; + --repo|-R) i=$((i+1)) ;; + --state=*) state="${a#--state=}" ;; + -s=*) state="${a#-s=}" ;; + -s?*) state="${a#-s}" ;; + --state|-s) i=$((i+1)); state="${args[$i]:-}" ;; + --limit=*) limit="${a#--limit=}" ;; + -L=*) limit="${a#-L=}" ;; + -L?*) limit="${a#-L}" ;; + --limit|-L) i=$((i+1)); limit="${args[$i]:-}" ;; + # gh accumulates repeated --json values, so every occurrence must be + # validated, not just the last one. + --json=*) json="${json:+${json},}${a#--json=}" ;; + --json) i=$((i+1)); json="${json:+${json},}${args[$i]:-}" ;; + -*) return 1 ;; + pr) [[ "$seen_sub" = 0 ]] || return 1; seen_sub=1 ;; + list) [[ "$seen_sub" = 1 && "$seen_act" = 0 ]] || return 1; seen_act=1 ;; + *) return 1 ;; + esac + done + [[ "$(_lower "$state")" = "open" ]] || return 1 + if [[ -n "$limit" ]]; then + [[ "$limit" =~ ^[0-9]+$ && "$limit" -le 100 ]] || return 1 + fi + [[ -n "$json" ]] || return 1 + local f _fields + IFS=',' read -ra _fields <<<"$json" + for f in "${_fields[@]}"; do + case "$f" in + number|headRefOid|isCrossRepository|statusCheckRollup) ;; + *) return 1 ;; + esac + done + return 0 +} + # Resolve the authenticated GitHub login. Initialize the cache internally so a # caller-controlled environment cannot seed a trusted identity (#3982). # @@ -726,6 +781,8 @@ if [ "$_gh_author_scoped" = true ]; then : elif _pr_list_merged_only; then : # Merged PRs only: history, not a work queue (see _pr_list_merged_only). + elif _pr_list_baseline_check; then + : # hive-baseline-check.sh's sibling lookup (see _pr_list_baseline_check). else # Root-caused in a live hive (2026-08-20): agents read this two-line # message as "all gh $subcmd commands are blocked" and silently skipped diff --git a/bin/gh-wrapper.test.sh b/bin/gh-wrapper.test.sh index 4435be94f1..cfd5dd43ce 100755 --- a/bin/gh-wrapper.test.sh +++ b/bin/gh-wrapper.test.sh @@ -524,6 +524,28 @@ _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state merged -S review (block pr list --repo test/repo --state merged -S review _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state merged --search=x (blocked: equals search flag)" \ pr list --repo test/repo --state merged --search=x +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 0 "-" "$ENUM_DENY" "pr list --state open statusCheckRollup with HIVE_BASELINE_CHECK=1 (allowed: baseline helper's sibling lookup)" \ + pr list --repo test/repo --state open --limit 100 --json number,headRefOid,isCrossRepository,statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 0 "-" "$ENUM_DENY" "pr list -R before the subcommand, equals/short forms with HIVE_BASELINE_CHECK=1 (allowed: same projection)" \ + -R test/repo pr list --state=open -L50 --json=number,statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state open --search with HIVE_BASELINE_CHECK=1 (blocked)" \ + pr list --repo test/repo --state open --search foo +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --label actionable with HIVE_BASELINE_CHECK=1 (blocked: flag outside the allowlist)" \ + pr list --repo test/repo --state open --label actionable --limit 100 --json number,headRefOid,isCrossRepository,statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --json number,title,statusCheckRollup with HIVE_BASELINE_CHECK=1 (blocked: field outside the projection)" \ + pr list --repo test/repo --state open --limit 100 --json number,title,statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --json title --json number with HIVE_BASELINE_CHECK=1 (blocked: gh accumulates repeated --json, every occurrence is checked)" \ + pr list --repo test/repo --state open --limit 100 --json title --json number +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 0 "-" "$ENUM_DENY" "pr list --json number --json statusCheckRollup with HIVE_BASELINE_CHECK=1 (allowed: repeated --json, all fields in the projection)" \ + pr list --repo test/repo --state open --limit 100 --json number --json statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --limit 500 with HIVE_BASELINE_CHECK=1 (blocked: over the helper's 100)" \ + pr list --repo test/repo --state open --limit 500 --json number,headRefOid,isCrossRepository,statusCheckRollup +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state open without --json with HIVE_BASELINE_CHECK=1 (blocked: no projection)" \ + pr list --repo test/repo --state open --limit 100 +HIVE_BASELINE_CHECK=1 _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state all with HIVE_BASELINE_CHECK=1 (blocked: --state open required)" \ + pr list --repo test/repo --state all --limit 100 --json number,statusCheckRollup +_run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state open --json statusCheckRollup without HIVE_BASELINE_CHECK (blocked)" \ + pr list --repo test/repo --state open --json statusCheckRollup _run_test_staff_gate 1 "$ENUM_DENY" "-" "pr list --state with no value (blocked)" \ pr list --repo test/repo --state _run_test_staff_gate 1 "$ENUM_DENY" "-" "issue list --state merged (blocked: issues are never merged)" \ diff --git a/changelog.d/added-17986-review-perspectives-source-config.md b/changelog.d/added-17986-review-perspectives-source-config.md new file mode 100644 index 0000000000..5965ebd30b --- /dev/null +++ b/changelog.d/added-17986-review-perspectives-source-config.md @@ -0,0 +1 @@ +- **`review.perspectives_source` config key and `kind: ReviewPerspectives` document format** for live-linking a hive's review perspective names and focus text to a file in a GitHub repo, so maintainers can change the set through ordinary PRs instead of hive config edits. This lands the config surface, a runtime-only resolved slot that is never written back to the overlay, and the validating document parser (`pkg/perspsrc`); fetching and wiring follow in later changes, so behaviour is unchanged until then and unchanged whenever the key is absent. Once resolved, the linked set wins over inline `review.perspectives` / `review.perspective_prompts`, which stay the fallback. ([#17986](https://github.com/hivecommons/hive/issues/17986), part of [#17975](https://github.com/hivecommons/hive/issues/17975)) diff --git a/changelog.d/fixed-signed-commit-skip-selfreport.md b/changelog.d/fixed-signed-commit-skip-selfreport.md new file mode 100644 index 0000000000..cbf6f24ade --- /dev/null +++ b/changelog.d/fixed-signed-commit-skip-selfreport.md @@ -0,0 +1 @@ +- The signed-commit reconciler's intentional refusal to rewrite a foreign or merge commit no longer files repeated unclassified-error fleet reports; its warning and PR remediation remain visible ([#17924](https://github.com/hivecommons/hive/issues/17924)). diff --git a/changelog.d/fixed-sse-drop-warn-flood.md b/changelog.d/fixed-sse-drop-warn-flood.md new file mode 100644 index 0000000000..a5494ff99b --- /dev/null +++ b/changelog.d/fixed-sse-drop-warn-flood.md @@ -0,0 +1 @@ +- Dashboard logs one summary warning per five minutes for dropped SSE events instead of one warning per event, so a stalled browser tab no longer triggers repeated-error fleet reports (#17995). diff --git a/dashboard/openapi.json b/dashboard/openapi.json index d3ed3532f2..3cfc4e3108 100644 --- a/dashboard/openapi.json +++ b/dashboard/openapi.json @@ -40515,6 +40515,1272 @@ } } } + }, + "/api/knowledge/connectors/{name}/auth": { + "get": { + "tags": [ + "Knowledge" + ], + "summary": "Get a knowledge connector's sign-in state", + "description": "Returns the cached sign-in state of a connector that authenticates interactively (e.g. `plaud` via the Plaud MCP OAuth flow). Pass `refresh=1` to re-query the upstream account. Tokens are never returned; they stay in the connector's own credential store.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Connector name" + }, + { + "name": "refresh", + "in": "query", + "required": false, + "schema": { + "type": "string", + "enum": [ + "1", + "true" + ] + }, + "description": "Re-query the upstream account instead of returning the cached state" + } + ], + "responses": { + "200": { + "description": "Sign-in state", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Interactive sign-in state of a connector that authenticates through its own OAuth flow (currently `plaud`).", + "properties": { + "connected": { + "type": "boolean", + "description": "Upstream reports a signed-in account" + }, + "account": { + "type": "string", + "description": "Signed-in identity (email or display name), if known" + }, + "pending": { + "type": "boolean", + "description": "A login is waiting for the user to authorize in the browser" + }, + "auth_url": { + "type": "string", + "format": "uri", + "description": "Authorization URL to open while pending" + }, + "message": { + "type": "string", + "description": "Last human-readable login/logout outcome" + }, + "checked_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "400": { + "description": "Connector type does not use interactive sign-in", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown connector, or no connectors configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/knowledge/connectors/{name}/auth/connect": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Start a knowledge connector's interactive sign-in", + "description": "Owner-only. Starts the connector's OAuth login (for `plaud`, the MCP server's `login` tool) and returns the pending state including `auth_url` for the user to open in a browser. The login session is kept alive for the callback; poll `GET /api/knowledge/connectors/{name}/auth?refresh=1` for the outcome.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Connector name" + } + ], + "responses": { + "202": { + "description": "Login started (or already connected)", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Interactive sign-in state of a connector that authenticates through its own OAuth flow (currently `plaud`).", + "properties": { + "connected": { + "type": "boolean", + "description": "Upstream reports a signed-in account" + }, + "account": { + "type": "string", + "description": "Signed-in identity (email or display name), if known" + }, + "pending": { + "type": "boolean", + "description": "A login is waiting for the user to authorize in the browser" + }, + "auth_url": { + "type": "string", + "format": "uri", + "description": "Authorization URL to open while pending" + }, + "message": { + "type": "string", + "description": "Last human-readable login/logout outcome" + }, + "checked_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "400": { + "description": "Connector type does not use interactive sign-in", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown connector, or no connectors configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Owner access required", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "502": { + "description": "The connector's MCP server failed to start the login", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/knowledge/connectors/{name}/auth/disconnect": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Sign a knowledge connector out", + "description": "Owner-only. Signs the connector out upstream (for `plaud`, the MCP server's `logout` tool), which removes the token from the connector's own credential store, and returns the new state.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Connector name" + } + ], + "responses": { + "200": { + "description": "Signed out", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Interactive sign-in state of a connector that authenticates through its own OAuth flow (currently `plaud`).", + "properties": { + "connected": { + "type": "boolean", + "description": "Upstream reports a signed-in account" + }, + "account": { + "type": "string", + "description": "Signed-in identity (email or display name), if known" + }, + "pending": { + "type": "boolean", + "description": "A login is waiting for the user to authorize in the browser" + }, + "auth_url": { + "type": "string", + "format": "uri", + "description": "Authorization URL to open while pending" + }, + "message": { + "type": "string", + "description": "Last human-readable login/logout outcome" + }, + "checked_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "400": { + "description": "Connector type does not use interactive sign-in", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown connector, or no connectors configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Owner access required", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "502": { + "description": "The connector's MCP server failed to sign out", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "ok": { + "type": "boolean" + }, + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/inbox": { + "get": { + "tags": [ + "Knowledge" + ], + "summary": "List inbox items (voice recordings and dictations) with their triage state", + "description": "Items are the facts written by voice knowledge connectors (`plaud`, `wisprflow`), newest first, keyed by `(source, id)`, with per-destination routing state, source connection status, candidate repositories and the auto-routing settings.", + "responses": { + "200": { + "description": "Inbox listing", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "description": "An incoming voice recording or dictation. Every item is already a Knowledge fact; the inbox only tracks triage.", + "properties": { + "source": { + "type": "string", + "description": "Connector type (plaud, wisprflow, …)" + }, + "connector": { + "type": "string" + }, + "id": { + "type": "string", + "description": "Source item id (recording id)" + }, + "title": { + "type": "string" + }, + "excerpt": { + "type": "string" + }, + "recorded_at": { + "type": "string", + "format": "date-time" + }, + "duration_ms": { + "type": "integer" + }, + "tags": { + "type": "array", + "items": { + "type": "string" + } + }, + "fact_slug": { + "type": "string" + }, + "fact_path": { + "type": "string" + }, + "layer": { + "type": "string" + }, + "routes": { + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Per-destination routing state of an inbox item.", + "properties": { + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "filed", + "delegated", + "failed" + ] + }, + "repo": { + "type": "string", + "description": "Target repository for spek_rfc" + }, + "url": { + "type": "string", + "format": "uri", + "description": "RFC issue or dibs idea URL once filed" + }, + "children": { + "type": "array", + "items": { + "type": "string", + "format": "uri" + }, + "description": "Child issue URLs filed by spektacular" + }, + "error": { + "type": "string" + }, + "updated_at": { + "type": "string", + "format": "date-time" + } + } + } + }, + "dismissed": { + "type": "boolean", + "description": "Kept as a Knowledge fact only" + }, + "processed": { + "type": "boolean", + "description": "Dismissed or routed to at least one destination" + } + } + } + }, + "unprocessed": { + "type": "integer" + }, + "sources": { + "type": "array", + "items": { + "type": "object", + "properties": { + "type": { + "type": "string" + }, + "connector": { + "type": "string" + }, + "configured": { + "type": "boolean" + }, + "connected": { + "type": "boolean" + }, + "account": { + "type": "string" + } + } + } + }, + "repos": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Repositories this hive works on (RFC targets)" + }, + "settings": { + "type": "object", + "description": "Inbox auto-routing settings and destination readiness.", + "properties": { + "sources": { + "type": "array", + "items": { + "type": "string" + } + }, + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "title_prefix": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "destination": { + "type": "string", + "enum": [ + "spek_rfc", + "dibs", + "dismiss" + ] + }, + "repo": { + "type": "string" + } + } + } + }, + "spek_repo": { + "type": "string" + }, + "dibs_url": { + "type": "string" + }, + "dibs_token_set": { + "type": "boolean" + }, + "spek_ready": { + "type": "boolean" + } + } + }, + "error": { + "type": "string" + } + } + } + } + } + }, + "503": { + "description": "Inbox not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/inbox/route": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Route an inbox item to a destination", + "description": "Owner-only. `spek_rfc` runs `spektacular voice --publish` inside the hive (or hands a prompt to the configured spoke agent) and files one RFC issue plus child issues on `repo` (default `inbox.spek.repo`); `dibs` posts the recording as a private draft idea to the configured dibs instance; `dismiss` keeps the item as a Knowledge fact only. Returns 202 with the queued state; poll `GET /api/inbox` for completion.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "source", + "item_id", + "destination" + ], + "properties": { + "source": { + "type": "string" + }, + "item_id": { + "type": "string" + }, + "destination": { + "type": "string", + "enum": [ + "spek_rfc", + "dibs", + "dismiss" + ] + }, + "repo": { + "type": "string", + "description": "owner/repo for spek_rfc" + } + } + } + } + } + }, + "responses": { + "202": { + "description": "Route accepted", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "item_id": { + "type": "string" + }, + "destination": { + "type": "string" + }, + "state": { + "type": "object", + "description": "Per-destination routing state of an inbox item.", + "properties": { + "status": { + "type": "string", + "enum": [ + "queued", + "running", + "filed", + "delegated", + "failed" + ] + }, + "repo": { + "type": "string", + "description": "Target repository for spek_rfc" + }, + "url": { + "type": "string", + "format": "uri", + "description": "RFC issue or dibs idea URL once filed" + }, + "children": { + "type": "array", + "items": { + "type": "string", + "format": "uri" + }, + "description": "Child issue URLs filed by spektacular" + }, + "error": { + "type": "string" + }, + "updated_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } + } + } + }, + "400": { + "description": "Invalid request, unknown destination, missing repo, or destination not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not the owner", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown item", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "409": { + "description": "Route already queued or running", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "503": { + "description": "Inbox not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/inbox/dismiss": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Mark inbox items as Knowledge only (or undo)", + "description": "Owner-only bulk dismiss. Dismissed items stay searchable as Knowledge facts and leave the unprocessed count; `undo: true` returns them to the inbox.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "required": [ + "items" + ], + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "item_id": { + "type": "string" + } + } + } + }, + "undo": { + "type": "boolean" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Items updated", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "dismissed": { + "type": "integer" + } + } + } + } + } + }, + "400": { + "description": "Invalid request", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not the owner", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "503": { + "description": "Inbox not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/inbox/settings": { + "get": { + "tags": [ + "Knowledge" + ], + "summary": "Read inbox auto-routing settings", + "responses": { + "200": { + "description": "Settings", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Inbox auto-routing settings and destination readiness.", + "properties": { + "sources": { + "type": "array", + "items": { + "type": "string" + } + }, + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "title_prefix": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "destination": { + "type": "string", + "enum": [ + "spek_rfc", + "dibs", + "dismiss" + ] + }, + "repo": { + "type": "string" + } + } + } + }, + "spek_repo": { + "type": "string" + }, + "dibs_url": { + "type": "string" + }, + "dibs_token_set": { + "type": "boolean" + }, + "spek_ready": { + "type": "boolean" + } + } + } + } + } + }, + "503": { + "description": "Inbox not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + }, + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Update inbox auto-routing settings", + "description": "Owner-only. Persists `inbox.rules`, `inbox.spek.repo` and `inbox.dibs.url` to the runtime config; omitted fields are left unchanged. Rules apply to items that arrive after the change.", + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "title_prefix": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "destination": { + "type": "string", + "enum": [ + "spek_rfc", + "dibs", + "dismiss" + ] + }, + "repo": { + "type": "string" + } + } + } + }, + "spek_repo": { + "type": "string" + }, + "dibs_url": { + "type": "string" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Settings saved", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Inbox auto-routing settings and destination readiness.", + "properties": { + "sources": { + "type": "array", + "items": { + "type": "string" + } + }, + "rules": { + "type": "array", + "items": { + "type": "object", + "properties": { + "source": { + "type": "string" + }, + "title_prefix": { + "type": "string" + }, + "tag": { + "type": "string" + }, + "destination": { + "type": "string", + "enum": [ + "spek_rfc", + "dibs", + "dismiss" + ] + }, + "repo": { + "type": "string" + } + } + } + }, + "spek_repo": { + "type": "string" + }, + "dibs_url": { + "type": "string" + }, + "dibs_token_set": { + "type": "boolean" + }, + "spek_ready": { + "type": "boolean" + } + } + } + } + } + }, + "400": { + "description": "Invalid rule or repository", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not the owner", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "503": { + "description": "Inbox not configured", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } + }, + "/api/knowledge/connectors/{name}/auth/complete": { + "post": { + "tags": [ + "Knowledge" + ], + "summary": "Finish a connector sign-in from pasted material (hosted hives)", + "description": "Owner-only. Plaud's OAuth client only redirects to `localhost:8199`, which a hive running in a cluster never receives. The operator pastes either the `redirect_url` the browser landed on after approving (forwarded to the in-pod callback while the login started by `/auth/connect` is still pending, 10 minutes) or the `tokens` JSON from a laptop's `~/.plaud/tokens-mcp.json`; the hive stores it and re-checks the account.", + "parameters": [ + { + "name": "name", + "in": "path", + "required": true, + "schema": { + "type": "string" + }, + "description": "Connector name" + } + ], + "requestBody": { + "required": true, + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "redirect_url": { + "type": "string", + "format": "uri" + }, + "tokens": { + "type": "string", + "description": "Raw JSON contents of tokens-mcp.json" + } + } + } + } + } + }, + "responses": { + "200": { + "description": "Sign-in state after completion", + "content": { + "application/json": { + "schema": { + "type": "object", + "description": "Interactive sign-in state of a connector that authenticates through its own OAuth flow (currently `plaud`).", + "properties": { + "connected": { + "type": "boolean", + "description": "Upstream reports a signed-in account" + }, + "account": { + "type": "string", + "description": "Signed-in identity (email or display name), if known" + }, + "pending": { + "type": "boolean", + "description": "A login is waiting for the user to authorize in the browser" + }, + "auth_url": { + "type": "string", + "format": "uri", + "description": "Authorization URL to open while pending" + }, + "message": { + "type": "string", + "description": "Last human-readable login/logout outcome" + }, + "checked_at": { + "type": "string", + "format": "date-time" + } + } + } + } + } + }, + "400": { + "description": "Connector does not support pasted completion, bad body, or no pending login", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "403": { + "description": "Not the owner", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "404": { + "description": "Unknown connector", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + }, + "502": { + "description": "Upstream rejected the material", + "content": { + "application/json": { + "schema": { + "type": "object", + "properties": { + "error": { + "type": "string" + } + } + } + } + } + } + } + } } }, "components": { diff --git a/src/Dockerfile b/src/Dockerfile index a53d961110..20090bb380 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -412,9 +412,9 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ # --- Layer 8.27: OMP CLI (Oh My Pi — backend: omp) --- # Mirrored from Dockerfile.contributor so hub agents can launch backend: omp. # Downloads are optional like Goose/agy, but checksum mismatches are fatal. -ARG OMP_VERSION=18.18.0 -ARG OMP_SHA256_AMD64=ec271e0d7df4c0f57acb6a7a28f8585d46bd050bcef8494f6d33db8a6c0dfbba -ARG OMP_SHA256_ARM64=b394bfbf7c0e0ba82e01913defb5fadf14a6ea5a71ee05348070ab5e40187144 +ARG OMP_VERSION=18.19.0 +ARG OMP_SHA256_AMD64=b44fda2ae5c9cce2bbe124d1672cba532a136d9e1eef68599b16efcd3bc5c626 +ARG OMP_SHA256_ARM64=61826bf89c2c0512417fee05fa2f486e60ee11b23ea95a17257a6c49edb6b8d5 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ && case "$ARCH" in \ amd64|x86_64) OMP_ASSET=omp-linux-x64; OMP_SHA256="$OMP_SHA256_AMD64" ;; \ diff --git a/src/Dockerfile.contributor b/src/Dockerfile.contributor index 361a635993..d45cd6f111 100644 --- a/src/Dockerfile.contributor +++ b/src/Dockerfile.contributor @@ -489,9 +489,9 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ # No credential is baked in: omp reads the selected model's provider # credential from its documented environment variables (ANTHROPIC_API_KEY, # OPENAI_API_KEY, …), which the Justfile's container arm already forwards. -ARG OMP_VERSION=18.18.0 -ARG OMP_SHA256_AMD64=ec271e0d7df4c0f57acb6a7a28f8585d46bd050bcef8494f6d33db8a6c0dfbba -ARG OMP_SHA256_ARM64=b394bfbf7c0e0ba82e01913defb5fadf14a6ea5a71ee05348070ab5e40187144 +ARG OMP_VERSION=18.19.0 +ARG OMP_SHA256_AMD64=b44fda2ae5c9cce2bbe124d1672cba532a136d9e1eef68599b16efcd3bc5c626 +ARG OMP_SHA256_ARM64=61826bf89c2c0512417fee05fa2f486e60ee11b23ea95a17257a6c49edb6b8d5 RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" \ && case "$ARCH" in \ amd64) OMP_ASSET=omp-linux-x64; OMP_SHA256="$OMP_SHA256_AMD64" ;; \ diff --git a/src/cmd/hive/boot.go b/src/cmd/hive/boot.go index ed5da1097f..fe4c8b2f07 100644 --- a/src/cmd/hive/boot.go +++ b/src/cmd/hive/boot.go @@ -29,6 +29,7 @@ import ( "github.com/hivecommons/hive/pkg/github/automerge" "github.com/hivecommons/hive/pkg/governor" "github.com/hivecommons/hive/pkg/hub/spoke" + "github.com/hivecommons/hive/pkg/inbox" "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/knowledge/connector" "github.com/hivecommons/hive/pkg/mention" @@ -237,6 +238,7 @@ type boot struct { // bootKnowledge gitSyncer *knowledge.GitSyncer knowledgeConnectors *connector.Syncer + inboxSvc *inbox.Service knowledgePublish *connector.Mirror promotionScheduler *knowledge.PromotionScheduler quotaAccount string diff --git a/src/cmd/hive/boot_dashboard_api_deps.go b/src/cmd/hive/boot_dashboard_api_deps.go index 0d99bb23ee..b173ec14de 100644 --- a/src/cmd/hive/boot_dashboard_api_deps.go +++ b/src/cmd/hive/boot_dashboard_api_deps.go @@ -2,9 +2,11 @@ package main import ( "context" + "log/slog" "time" "github.com/hivecommons/hive/pkg/dashboard" + "github.com/hivecommons/hive/pkg/github" ) // bootDashboardAPIDeps are the long-lived effects bootDashboardAPI performs @@ -22,13 +24,12 @@ import ( type bootDashboardAPIDeps struct { registerAPI func(srv *dashboard.Server, d *dashboard.Dependencies) setForgeAppInventory func(srv *dashboard.Server, fn func() dashboard.ForgeAppInventory) + classifyRepoCoverage func(context.Context, *github.AppAuth, string, []string, *slog.Logger) (bool, string, github.AppAuthState) // startInstallDiscovery runs tryDiscover once immediately and then every // githubAppDiscoveryInterval until ctx is done. startInstallDiscovery func(ctx context.Context, tryDiscover func()) // startSelfHeal runs heal every githubAppSelfHealInterval until ctx is - // done. It does NOT run immediately: boot has just applied the classified - // App state and the first tick is cheap because heal no-ops while the - // banner is not showing. + // done. It does NOT run immediately: boot has just checked coverage. startSelfHeal func(ctx context.Context, heal func()) startInceptionWatcher func(ctx context.Context, w *dashboard.InceptionWatcher) } @@ -39,15 +40,13 @@ type bootDashboardAPIDeps struct { // requiring anyone to paste it. const githubAppDiscoveryInterval = 5 * time.Minute -// githubAppSelfHealInterval mirrors the heartbeat cadence so a stale banner -// clears within one heartbeat window of the app becoming healthy, without -// adding meaningful GitHub API load (the check only runs while the banner is -// actually showing). +// githubAppSelfHealInterval paces both coverage checks and banner healing. const githubAppSelfHealInterval = 2 * time.Minute func defaultBootDashboardAPIDeps() bootDashboardAPIDeps { return bootDashboardAPIDeps{ - registerAPI: func(srv *dashboard.Server, d *dashboard.Dependencies) { srv.RegisterAPI(d) }, + classifyRepoCoverage: classifyGitHubAppRepoCoverage, + registerAPI: func(srv *dashboard.Server, d *dashboard.Dependencies) { srv.RegisterAPI(d) }, setForgeAppInventory: func(srv *dashboard.Server, fn func() dashboard.ForgeAppInventory) { srv.SetForgeAppInventoryFn(fn) }, diff --git a/src/cmd/hive/boot_dashboard_api_deps_test.go b/src/cmd/hive/boot_dashboard_api_deps_test.go index 076ef3cdc0..bea7683d49 100644 --- a/src/cmd/hive/boot_dashboard_api_deps_test.go +++ b/src/cmd/hive/boot_dashboard_api_deps_test.go @@ -37,6 +37,9 @@ type bootDashboardAPIFake struct { func newBootDashboardAPIFake() *bootDashboardAPIFake { f := &bootDashboardAPIFake{} f.deps = bootDashboardAPIDeps{ + classifyRepoCoverage: func(context.Context, *github.AppAuth, string, []string, *slog.Logger) (bool, string, github.AppAuthState) { + return false, "", github.AppStateUnknown + }, registerAPI: func(srv *dashboard.Server, d *dashboard.Dependencies) { f.registeredOn, f.registered = srv, d }, @@ -221,6 +224,39 @@ func TestBootDashboardAPIWith_TickBodiesAreCheapWhenNothingToDo(t *testing.T) { } } +func TestSelfHealDetectsCoverageWithoutExistingBanner(t *testing.T) { + for _, tc := range []struct { + name string + raise bool + state github.AppAuthState + }{ + {"missing", true, github.AppStateRepoNotCovered}, + {"transient", false, github.AppStateUnknown}, + {"covered", false, github.AppStateOK}, + } { + t.Run(tc.name, func(t *testing.T) { + f := newBootDashboardAPIFake() + cfg := bootDashboardAPIConfig() + cfg.Project.PrimaryRepo = "private" + b := newBootDashboardAPIBoot(t, f, cfg) + b.ghClient = fakeGitHubClient(t) + called := false + f.deps.classifyRepoCoverage = func(_ context.Context, _ *github.AppAuth, org string, repos []string, _ *slog.Logger) (bool, string, github.AppAuthState) { + called = true + if org != "acme" || len(repos) != 3 || repos[2] != "private" { + t.Fatalf("coverage inputs: %q %v", org, repos) + } + return tc.raise, "select private in the installation", tc.state + } + b.bootDashboardAPIWith(f.deps) + f.healTick() + if !called || b.dashSrv.IsGitHubAppRequired() != tc.raise { + t.Fatalf("called=%v required=%v", called, b.dashSrv.IsGitHubAppRequired()) + } + }) + } +} + func TestBootDashboardAPIWith_ForgeInventoryResolvesTheActiveKey(t *testing.T) { f := newBootDashboardAPIFake() cfg := bootDashboardAPIConfig() diff --git a/src/cmd/hive/boot_github_deps.go b/src/cmd/hive/boot_github_deps.go index 1dbba48eb0..3f976f0c80 100644 --- a/src/cmd/hive/boot_github_deps.go +++ b/src/cmd/hive/boot_github_deps.go @@ -26,8 +26,9 @@ func defaultBootGitHubDeps() bootGitHubDeps { // notifier, ACMM inference, attribution hook, and on-disk default policy seed // run for real (cfg.Policies.LocalDir points a test at a temp dir). type bootAdvisoryDeps struct { - ensureAdvisoryIssue func(ctx context.Context, c *github.Client, repo string) (int, error) - classifyAppFailure func(ctx context.Context, appAuth *github.AppAuth, expectedOwner string, logger *slog.Logger) (bool, string, github.AppAuthState) + ensureAdvisoryIssue func(ctx context.Context, c *github.Client, repo string) (int, error) + classifyAppFailure func(ctx context.Context, appAuth *github.AppAuth, expectedOwner string, logger *slog.Logger) (bool, string, github.AppAuthState) + classifyRepoCoverage func(ctx context.Context, appAuth *github.AppAuth, org string, repos []string, logger *slog.Logger) (bool, string, github.AppAuthState) } func defaultBootAdvisoryDeps() bootAdvisoryDeps { @@ -35,6 +36,7 @@ func defaultBootAdvisoryDeps() bootAdvisoryDeps { ensureAdvisoryIssue: func(ctx context.Context, c *github.Client, repo string) (int, error) { return c.EnsureAdvisoryIssue(ctx, repo) }, - classifyAppFailure: classifyGitHubAppFailure, + classifyAppFailure: classifyGitHubAppFailure, + classifyRepoCoverage: classifyGitHubAppRepoCoverage, } } diff --git a/src/cmd/hive/boot_github_deps_test.go b/src/cmd/hive/boot_github_deps_test.go index 438b34332c..1dc2faa30d 100644 --- a/src/cmd/hive/boot_github_deps_test.go +++ b/src/cmd/hive/boot_github_deps_test.go @@ -9,6 +9,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/github" @@ -97,6 +98,9 @@ type bootAdvisoryFake struct { func newBootAdvisoryFake() *bootAdvisoryFake { f := &bootAdvisoryFake{ensureNum: 42} f.deps = bootAdvisoryDeps{ + classifyRepoCoverage: func(context.Context, *github.AppAuth, string, []string, *slog.Logger) (bool, string, github.AppAuthState) { + return false, "", github.AppStateUnknown + }, ensureAdvisoryIssue: func(_ context.Context, _ *github.Client, repo string) (int, error) { f.ensured = append(f.ensured, repo) return f.ensureNum, f.ensureErr @@ -121,6 +125,52 @@ func bootAdvisoryConfig(t *testing.T) *config.Config { return cfg } +func TestBootAdvisoryDetectsMissingRepoCoverage(t *testing.T) { + for _, tc := range []struct { + name string + raise bool + state github.AppAuthState + }{ + {"missing", true, github.AppStateRepoNotCovered}, + {"transient", false, github.AppStateUnknown}, + {"covered", false, github.AppStateOK}, + } { + t.Run(tc.name, func(t *testing.T) { + cfg := bootAdvisoryConfig(t) + cfg.Project.PrimaryRepo = "private" + level := 0 + cfg.ACMMLevel = &level + b, _ := newDepsTestBoot(t, cfg) + b.ghClient = fakeGitHubClient(t) + f := newBootAdvisoryFake() + called := false + var coverageCtx context.Context + f.deps.classifyRepoCoverage = func(ctx context.Context, _ *github.AppAuth, org string, repos []string, _ *slog.Logger) (bool, string, github.AppAuthState) { + called = true + coverageCtx = ctx + deadline, ok := ctx.Deadline() + if !ok || time.Until(deadline) > advisoryEnsureBudget { + t.Fatalf("coverage deadline = %v, present=%v", deadline, ok) + } + if org != "acme" || len(repos) != 3 || repos[2] != "private" { + t.Fatalf("coverage inputs: %q %v", org, repos) + } + return tc.raise, "select private in the installation", tc.state + } + b.bootAdvisoryWith(f.deps) + if !called || b.githubAppRequired != tc.raise { + t.Fatalf("called=%v required=%v", called, b.githubAppRequired) + } + if coverageCtx.Err() != context.Canceled { + t.Fatalf("coverage context not canceled after check: %v", coverageCtx.Err()) + } + if tc.raise && (b.githubAppState != tc.state || b.githubAppDiag != "select private in the installation") { + t.Fatalf("coverage diagnosis: %v %q", b.githubAppState, b.githubAppDiag) + } + }) + } +} + func TestBootAdvisoryWithEnsuresIssueAndWritesPolicy(t *testing.T) { t.Setenv("HIVE_ADVISORY_ISSUE", "") cfg := bootAdvisoryConfig(t) @@ -281,3 +331,29 @@ func TestBootAdvisoryWithRateLimitDoesNotClassify(t *testing.T) { t.Fatalf("log:\n%s", log.String()) } } + +func TestBootAdvisoryWithBoundsBlockedEnsure(t *testing.T) { + old := advisoryEnsureBudget + advisoryEnsureBudget = 50 * time.Millisecond + defer func() { advisoryEnsureBudget = old }() + + cfg := bootAdvisoryConfig(t) + b, _ := newDepsTestBoot(t, cfg) + b.ghClient = fakeGitHubClient(t) + f := newBootAdvisoryFake() + f.deps.ensureAdvisoryIssue = func(ctx context.Context, _ *github.Client, _ string) (int, error) { + <-ctx.Done() + return 0, ctx.Err() + } + + done := make(chan struct{}) + go func() { b.bootAdvisoryWith(f.deps); close(done) }() + select { + case <-done: + case <-time.After(5 * time.Second): + t.Fatal("bootAdvisory must not block on a hung advisory-issue ensure") + } + if len(b.advisoryIssues) != 0 { + t.Fatalf("advisoryIssues = %v", b.advisoryIssues) + } +} diff --git a/src/cmd/hive/boot_inbox.go b/src/cmd/hive/boot_inbox.go new file mode 100644 index 0000000000..d4a3e0808b --- /dev/null +++ b/src/cmd/hive/boot_inbox.go @@ -0,0 +1,105 @@ +package main + +import ( + "context" + "fmt" + "path/filepath" + "time" + + "github.com/hivecommons/hive/pkg/dashboard" + "github.com/hivecommons/hive/pkg/inbox" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// inboxSweepTimeout bounds the auto-routing sweep run after a connector sync. +const inboxSweepTimeout = 2 * time.Minute + +// bootInbox builds the Inbox service. Sources are the configured knowledge +// connectors whose type is a voice source (inbox.sources, default +// plaud/wisprflow); with none configured the service still exists and lists +// nothing, so the dashboard shows the empty state. Items are read from the connector vaults under /connectors; +// route state persists at /connector-state/_inbox.json (connector +// names cannot start with "_", so it never collides with a connector). +func (b *boot) bootInbox(baseDir string) { + sources := b.cfg.Inbox.EffectiveSources() + configured := map[string]bool{} + for _, c := range b.cfg.Knowledge.Connectors { + configured[c.Type] = true + } + var srcs []inbox.Source + for _, typ := range sources { + if configured[typ] { + srcs = append(srcs, b.inboxSource(baseDir, typ)) + } + } + token := func(ctx context.Context, _ string) (string, error) { + if b.appAuth == nil { + return "", nil + } + return b.appAuth.Token(ctx) + } + kick := func(agent, prompt string) error { + if b.agentMgr == nil { + return fmt.Errorf("agent runtime is not available") + } + started, err := b.agentMgr.SendKickAsync(agent, prompt) + if err != nil { + return err + } + if !started { + return fmt.Errorf("agent %s did not accept the kick", agent) + } + return nil + } + store := inbox.NewStore(filepath.Join(baseDir, "connector-state", "_inbox.json")) + svc := inbox.NewService(b.cfg.Inbox, store, srcs, dashboard.NewInboxDestinations(b.cfg, token, kick), b.logger) + svc.SetReadiness(dashboard.InboxReadiness(b.cfg)) + b.inboxSvc = svc + b.logger.Info("inbox started", "sources", len(srcs), "rules", len(b.cfg.Inbox.Rules)) +} + +// inboxSource returns the fact-backed source for typ. For Plaud it prefers +// reading recordings directly from the connector while it is signed in. +func (b *boot) inboxSource(baseDir, typ string) inbox.Source { + facts := &inbox.FactSource{ConnectorsDir: filepath.Join(baseDir, "connectors"), SourceType: typ} + if typ != inbox.PlaudType || b.knowledgeConnectors == nil { + return facts + } + for _, c := range b.cfg.Knowledge.Connectors { + if c.Type != typ { + continue + } + conn, ok := b.knowledgeConnectors.Connector(c.Name) + reader, isReader := conn.(connector.PlaudReader) + if !ok || !isReader { + continue + } + direct := &inbox.PlaudSource{Name: c.Name, Reader: reader} + if auth, ok := conn.(connector.Authenticator); ok { + direct.Ready = func(ctx context.Context) bool { return auth.AuthStatus(ctx, false).Connected } + } + return &inbox.PreferDirect{Direct: direct, Facts: facts} + } + return facts +} + +// inboxAfterSync runs the auto-routing rules once a voice connector synced. +func (b *boot) inboxAfterSync(st connector.Status) { + svc := b.inboxSvc + if svc == nil { + return + } + for _, typ := range b.cfg.Inbox.EffectiveSources() { + if typ != st.Type { + continue + } + go func() { + ctx, cancel := context.WithTimeout(context.Background(), inboxSweepTimeout) + defer cancel() + if n, err := svc.Sweep(ctx); err != nil { + b.logger.Warn("inbox sweep", "error", err, "routed", n) + } + }() + return + } +} diff --git a/src/cmd/hive/boot_inbox_test.go b/src/cmd/hive/boot_inbox_test.go new file mode 100644 index 0000000000..f1c38a836f --- /dev/null +++ b/src/cmd/hive/boot_inbox_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "context" + "io" + "log/slog" + "testing" + + "github.com/hivecommons/hive/pkg/config" +) + +// With no knowledge connectors configured the Inbox service must still boot +// so GET /api/inbox returns an empty list, not a 404 (#17918). +func TestBootInboxWithoutConnectors(t *testing.T) { + b := &boot{cfg: &config.Config{}, logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + b.bootKnowledgeConnectors(bootKnowledgeDeps{}) + if b.inboxSvc == nil { + t.Fatal("inbox service not built without connectors") + } + items, err := b.inboxSvc.Items(context.Background()) + if err != nil || len(items) != 0 { + t.Fatalf("items = %v %v", items, err) + } + if b.knowledgeConnectors != nil { + t.Fatal("connector syncer should stay nil with no connectors") + } +} diff --git a/src/cmd/hive/boot_knowledge_connectors.go b/src/cmd/hive/boot_knowledge_connectors.go index 0110c8a04b..070f0f9b5e 100644 --- a/src/cmd/hive/boot_knowledge_connectors.go +++ b/src/cmd/hive/boot_knowledge_connectors.go @@ -40,7 +40,7 @@ func connectorConfigs(entries []config.KnowledgeConnector) []connector.Connector // searchable like any other vault. Cursors and status persist under // /connector-state. A nil reg uses connector.DefaultRegistry(). // Returns nil when no connectors are configured. -func newKnowledgeConnectorSyncer(entries []config.KnowledgeConnector, reg *connector.Registry, api *knowledge.KnowledgeAPI, baseDir string, logger *slog.Logger) (*connector.Syncer, error) { +func newKnowledgeConnectorSyncer(entries []config.KnowledgeConnector, reg *connector.Registry, api *knowledge.KnowledgeAPI, baseDir string, logger *slog.Logger, afterSync func(connector.Status)) (*connector.Syncer, error) { if len(entries) == 0 { return nil, nil } @@ -79,6 +79,9 @@ func newKnowledgeConnectorSyncer(entries []config.KnowledgeConnector, reg *conne if store := api.GetVaultStore(filepath.Join(baseDir, "connectors", st.Layer)); store != nil { store.Reindex() } + if afterSync != nil { + afterSync(st) + } }, }) } @@ -86,7 +89,11 @@ func newKnowledgeConnectorSyncer(entries []config.KnowledgeConnector, reg *conne // bootKnowledgeConnectors builds and starts the connector syncer. A failure // is logged and leaves connectors off; it never blocks the rest of boot. func (b *boot) bootKnowledgeConnectors(deps bootKnowledgeDeps) { - s, err := newKnowledgeConnectorSyncer(b.cfg.Knowledge.Connectors, nil, b.knowledgeAPI, knowledge.BaseDir(), b.logger) + // The Inbox always boots, even with no connectors, so GET /api/inbox + // answers 200 with an empty list and the panel renders its empty state + // (#17918) instead of waiting forever on a 404. + b.bootInbox(knowledge.BaseDir()) + s, err := newKnowledgeConnectorSyncer(b.cfg.Knowledge.Connectors, nil, b.knowledgeAPI, knowledge.BaseDir(), b.logger, b.inboxAfterSync) if err != nil { b.logger.Warn("knowledge connectors disabled", "error", err) return diff --git a/src/cmd/hive/boot_knowledge_connectors_test.go b/src/cmd/hive/boot_knowledge_connectors_test.go index 048b2ea905..b0ca37dcaa 100644 --- a/src/cmd/hive/boot_knowledge_connectors_test.go +++ b/src/cmd/hive/boot_knowledge_connectors_test.go @@ -44,7 +44,7 @@ func (bootFakeConnector) Sync(_ context.Context, _ connector.Cursor, emit func(c func TestNewKnowledgeConnectorSyncer(t *testing.T) { logger := slog.New(slog.NewTextHandler(io.Discard, nil)) - if s, err := newKnowledgeConnectorSyncer(nil, nil, nil, t.TempDir(), logger); s != nil || err != nil { + if s, err := newKnowledgeConnectorSyncer(nil, nil, nil, t.TempDir(), logger, nil); s != nil || err != nil { t.Fatalf("no connectors = %v, %v; want nil, nil", s, err) } @@ -58,7 +58,7 @@ func TestNewKnowledgeConnectorSyncer(t *testing.T) { api := knowledge.NewKnowledgeAPI(nil, knowledge.KnowledgeConfig{Enabled: true, Engine: "file"}, logger) s, err := newKnowledgeConnectorSyncer([]config.KnowledgeConnector{ {Name: "guide", Type: "fake", Layer: "project"}, - }, reg, api, base, logger) + }, reg, api, base, logger, nil) if err != nil || s == nil { t.Fatalf("newKnowledgeConnectorSyncer = %v, %v", s, err) } diff --git a/src/cmd/hive/claimswire.go b/src/cmd/hive/claimswire.go index 9f3aedc6b1..a4b1452da2 100644 --- a/src/cmd/hive/claimswire.go +++ b/src/cmd/hive/claimswire.go @@ -93,7 +93,7 @@ func retryClaimEffect(ctx context.Context, logger *slog.Logger, what, key string return n } if logger != nil { - logger.Warn("issue-claims: retrying GitHub side effect", "what", what, "issue", key, "attempt", n, "retry_in", backoff.String(), "error", err) + logger.Info("issue-claims: retrying GitHub side effect", "what", what, "issue", key, "attempt", n, "retry_in", backoff.String(), "error", err) } if !wait(ctx, backoff) { if logger != nil { diff --git a/src/cmd/hive/claimswire_retry_test.go b/src/cmd/hive/claimswire_retry_test.go index 3792af3928..b3ac7d9559 100644 --- a/src/cmd/hive/claimswire_retry_test.go +++ b/src/cmd/hive/claimswire_retry_test.go @@ -1,9 +1,12 @@ package main import ( + "bytes" "context" "errors" + "log/slog" "net/http" + "strings" "testing" "time" @@ -138,3 +141,23 @@ func TestClaimEffectRetryable(t *testing.T) { } } } + +// An expected transient retry is logged below WARN so the fleet-report +// classifier does not file it; only exhaustion warns, once +// (hivecommons/hive#17782). +func TestRetryClaimEffect_RetryLoggedBelowWarn(t *testing.T) { + swapClaimsEffectRetry(t, 3, nil) + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, nil)) + retryClaimEffect(context.Background(), logger, "claimed stage", "acme/widgets#7", func() error { + return mutation.ErrNeedsReconciliation + }) + for _, line := range strings.Split(strings.TrimSpace(buf.String()), "\n") { + if strings.Contains(line, "retrying GitHub side effect") && strings.Contains(line, "level=WARN") { + t.Fatalf("retry logged at WARN: %s", line) + } + } + if got := strings.Count(buf.String(), "GitHub side effect failed"); got != 1 { + t.Fatalf("exhaustion warnings = %d, want 1", got) + } +} diff --git a/src/cmd/hive/eval_cycle_run_7232_test.go b/src/cmd/hive/eval_cycle_run_7232_test.go index b3f6dd5371..64d2569c45 100644 --- a/src/cmd/hive/eval_cycle_run_7232_test.go +++ b/src/cmd/hive/eval_cycle_run_7232_test.go @@ -6,6 +6,7 @@ import ( "net/http" "net/http/httptest" "strings" + "sync" "sync/atomic" "testing" "time" @@ -25,10 +26,12 @@ import ( // touches on a quiet org: every list endpoint is empty, every object endpoint // is a minimal document, and nothing ever errors. It records the paths hit so // a test can assert the cycle actually enumerated rather than bailed early. -func newEvalCycleAPI(t *testing.T, hits *[]string) *httptest.Server { +func newEvalCycleAPI(t *testing.T, mu *sync.Mutex, hits *[]string) *httptest.Server { t.Helper() srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() *hits = append(*hits, r.Method+" "+r.URL.Path) + mu.Unlock() w.Header().Set("Content-Type", "application/json") switch { case r.URL.Path == "/rate_limit": @@ -63,6 +66,7 @@ type evalCycleFixture struct { dashSrv *dashboard.Server last *atomic.Pointer[github.ActionableResult] hits []string + hitsMu sync.Mutex beadStores map[string]*beads.Store advisoryIssues map[string]int @@ -70,8 +74,11 @@ type evalCycleFixture struct { func newEvalCycleFixture(t *testing.T) *evalCycleFixture { t.Helper() + // A prior rate-limit test can leave process-wide request pacing enabled. + github.ResetRateLimitPacingForTest() + t.Cleanup(github.ResetRateLimitPacingForTest) f := &evalCycleFixture{last: new(atomic.Pointer[github.ActionableResult])} - api := newEvalCycleAPI(t, &f.hits) + api := newEvalCycleAPI(t, &f.hitsMu, &f.hits) logger := restoreTestLogger() f.cfg = &config.Config{} @@ -92,7 +99,8 @@ func newEvalCycleFixture(t *testing.T) *evalCycleFixture { func (f *evalCycleFixture) run(t *testing.T) { t.Helper() - ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) + // Bound a stalled in-process cycle without imposing a tight CI deadline. + ctx, cancel := context.WithTimeout(t.Context(), 60*time.Second) defer cancel() runEvalCycle(ctx, f.cfg, f.gh, f.gov, f.sched, f.agentMgr, f.dashSrv, nil, // notifier @@ -129,7 +137,10 @@ func TestRunEvalCycle_QuietOrgEnumeratesAndPublishesStatus(t *testing.T) { f.run(t) - if len(f.hits) == 0 { + f.hitsMu.Lock() + hitCount := len(f.hits) + f.hitsMu.Unlock() + if hitCount == 0 { t.Fatal("runEvalCycle made no GitHub calls: it bailed before enumeration") } if got := f.last.Load(); got == nil { @@ -177,14 +188,17 @@ func TestRunEvalCycle_PostsAdvisoryDigestAndHealsAppAuthFinding(t *testing.T) { f.run(t) + f.hitsMu.Lock() + hits := append([]string(nil), f.hits...) + f.hitsMu.Unlock() var posted bool - for _, h := range f.hits { + for _, h := range hits { if h == "POST /repos/testorg/widget/issues/4/comments" { posted = true } } if !posted { - t.Fatalf("digest was never posted to the pinned issue; hits=%v", f.hits) + t.Fatalf("digest was never posted to the pinned issue; hits=%v", hits) } healed, err := store.Get(finding.ID) if err != nil { diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index f431256ae4..95a59c3843 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -1165,6 +1165,7 @@ func (b *boot) wireBootClosures() { BeadSynthesizer: b.beadSynth, KnowledgeConnectors: b.knowledgeConnectorRuntime(), KnowledgePublish: b.knowledgePublishRuntime(), + Inbox: b.inboxSvc, BeadStores: b.beadStores, BeadStoreLoadFailures: b.beadStoreLoadFailures, AuditLedger: auditLedger, @@ -1684,6 +1685,11 @@ func (b *boot) newKnowledgePrimer() *knowledge.Primer { return p } +// advisoryEnsureBudget caps the startup advisory-issue ensure (and the failure +// classification after it) so a slow or unreachable GitHub cannot hold the +// dashboard bind past the kubelet liveness budget (#17827). +var advisoryEnsureBudget = 15 * time.Second + // bootAdvisory builds the notifier, infers the ACMM level, seeds the // GitHub App banner state, opens the mutation-convergence ledger and journal, // finds or creates the pinned advisory issue and seeds the embedded default @@ -1766,7 +1772,12 @@ func (b *boot) bootAdvisoryWith(deps bootAdvisoryDeps) bool { primaryRepo = b.cfg.Project.Repos[0] } if primaryRepo != "" { - num, err := deps.ensureAdvisoryIssue(b.ctx, b.ghClient, primaryRepo) + // Bounded: every GitHub App token mint against an unreachable + // GHE burns ~8s, and this runs before the dashboard binds. An + // unresolved issue is retried per eval cycle (ensurePinnedAdvisoryIssue). + ensureCtx, cancelEnsure := context.WithTimeout(b.ctx, advisoryEnsureBudget) + num, err := deps.ensureAdvisoryIssue(ensureCtx, b.ghClient, primaryRepo) + cancelEnsure() if err != nil { b.logger.Error("failed to ensure advisory issue", "repo", primaryRepo, "error", err) // GitHub returns 403 for rate limiting too — a transient @@ -1783,7 +1794,9 @@ func (b *boot) bootAdvisoryWith(deps bootAdvisoryDeps) bool { // vanished on the first Re-check with nothing fixed. // classifyGitHubAppFailure is the same verdict Re-check // uses, and it declines to raise on AppStateUnknown. - raise, diag, state := deps.classifyAppFailure(b.ctx, b.ghClient.AppAuth(), b.cfg.Project.Org, b.logger) + classifyCtx, cancelClassify := context.WithTimeout(b.ctx, advisoryEnsureBudget) + raise, diag, state := deps.classifyAppFailure(classifyCtx, b.ghClient.AppAuth(), b.cfg.Project.Org, b.logger) + cancelClassify() if raise { b.githubAppRequired = true b.githubAppDiag, b.githubAppState = diag, state @@ -1803,6 +1816,18 @@ func (b *boot) bootAdvisoryWith(deps bootAdvisoryDeps) bool { } } } + // Installation-level permissions do not establish that every configured + // repository is selected, including when advisory reads happen to succeed. + if b.ghClient != nil { + coverageCtx, cancelCoverage := context.WithTimeout(b.ctx, advisoryEnsureBudget) + raise, diag, state := deps.classifyRepoCoverage(coverageCtx, b.ghClient.AppAuth(), b.cfg.Project.Org, appCoverageRepos(b.cfg), b.logger) + cancelCoverage() + if raise { + b.githubAppRequired = true + b.githubAppDiag, b.githubAppState = diag, state + b.logger.Warn("GitHub App installation does not cover every configured repo", "state", state.String(), "detail", diag) + } + } b.advisoryStore = advisory.NewStore() @@ -3793,7 +3818,7 @@ func (b *boot) bootDashboardAPIWith(deps bootDashboardAPIDeps) { // as "app not installed / no read" — sending the operator after // credentials that were never broken. Checking first means the // specific, correct message wins over the generic one. - if raise, diag, state := classifyGitHubAppRepoCoverage(b.ctx, b.ghClient.AppAuth(), b.cfg.Project.Org, b.cfg.Project.Repos, b.logger); raise { + if raise, diag, state := deps.classifyRepoCoverage(b.ctx, b.ghClient.AppAuth(), b.cfg.Project.Org, appCoverageRepos(b.cfg), b.logger); raise { b.dashSrv.SetGitHubAppPermIssue(diag) b.dashSrv.SetGitHubAppState(state.String()) b.logger.Warn("github app recheck: installation does not cover every configured repo", @@ -3882,12 +3907,9 @@ func (b *boot) bootDashboardAPIWith(deps bootDashboardAPIDeps) { // so a later transient write failure that re-set the flag was never // re-evaluated — the banner stuck until the pod was restarted. // - // The loop therefore runs for the lifetime of the process (it does NOT - // return after the first success) and, whenever the banner is currently - // showing, re-runs the SAME read+write verification as the manual "Re-check" - // button (githubAppRecheckFn, which calls diagnoseGitHubApp) and clears - // the flag on success. When the banner is not showing there is nothing to do, - // so the tick is a cheap no-op that makes no GitHub API calls. + // The loop runs for the lifetime of the process. It detects selection + // changes even when no banner is showing, and re-runs the manual re-check + // while a banner is present. Inconclusive coverage checks never raise one. { primaryRepo := b.cfg.Project.PrimaryRepo if primaryRepo == "" && len(b.cfg.Project.Repos) > 0 { @@ -3895,7 +3917,15 @@ func (b *boot) bootDashboardAPIWith(deps bootDashboardAPIDeps) { } if primaryRepo != "" { deps.startSelfHeal(b.ctx, func() { - // Nothing to heal unless the banner is showing. + if !b.dashSrv.IsGitHubAppRequired() && b.ghClient != nil { + if raise, diag, state := deps.classifyRepoCoverage(b.ctx, b.ghClient.AppAuth(), b.cfg.Project.Org, appCoverageRepos(b.cfg), b.logger); raise { + b.dashSrv.SetGitHubAppRequired(true) + b.dashSrv.SetGitHubAppPermIssue(diag) + b.dashSrv.SetGitHubAppState(state.String()) + b.logger.Warn("github app self-heal: installation does not cover every configured repo", "state", state.String(), "detail", diag) + return + } + } if !b.dashSrv.IsGitHubAppRequired() { return } @@ -7720,7 +7750,9 @@ func parseColorInt(color string) int { // operator who flips the Security tab's sandbox toggle never restarts, so a // boot-only check would never reach the person who most needs it. func logAgentSandboxPosture(logger *slog.Logger, cfg *config.Config) { - for _, warning := range config.AgentSandboxGateWarnings(cfg) { + report := config.AgentSandboxGateReport(cfg) + config.StoreSandboxGateReport(report) + for _, warning := range report.Warnings { logger.Warn("agent sandbox posture", "warning", warning) } } diff --git a/src/cmd/hive/main_github_app.go b/src/cmd/hive/main_github_app.go index 5bb5fa99dd..b747bdca20 100644 --- a/src/cmd/hive/main_github_app.go +++ b/src/cmd/hive/main_github_app.go @@ -487,6 +487,22 @@ func classifyGitHubAppRepoCoverage(ctx context.Context, appAuth *github.AppAuth, return apphealth.ClassifyRepoCoverage(ctx, appAuth, org, repos, logger) } +// appCoverageRepos includes the primary repo even when it is not in the +// watched list; it is also used for advisory writes. +func appCoverageRepos(cfg *config.Config) []string { + repos := cfg.Project.Repos + primary := cfg.Project.PrimaryRepo + if primary == "" { + return repos + } + for _, repo := range repos { + if strings.EqualFold(repo, primary) || strings.EqualFold(repo, cfg.Project.Org+"/"+primary) || strings.EqualFold(cfg.Project.Org+"/"+repo, primary) { + return repos + } + } + return append(append([]string(nil), repos...), primary) +} + // The advisory digest posting policy moved to pkg/advisory (#7238 stage 2). // These wrappers keep the existing call sites unchanged; advisoryPostGate is // now an owned instance rather than a package-level struct tests reset. diff --git a/src/cmd/hive/review_kick_order.go b/src/cmd/hive/review_kick_order.go index ba5aa40aa0..a949df169b 100644 --- a/src/cmd/hive/review_kick_order.go +++ b/src/cmd/hive/review_kick_order.go @@ -4,6 +4,7 @@ import ( "fmt" "log/slog" "sync" + "time" "github.com/hivecommons/hive/pkg/review" "github.com/hivecommons/hive/pkg/scheduler" @@ -99,10 +100,20 @@ func orderReviewFixKicksFirst(messages []scheduler.KickMessage, reviewKickByMess // particular PR is being starved. The planner drops the undelivered kick // from the dispatch state (review.ConfirmDelivered) and re-plans it on the // next cycle, so the line says so. +// +// The same kick for the same PR head fails with the same cause every cycle +// while an agent's pane is not ready, so repeats are logged at debug (and +// not re-audited) until the cause changes or undeliveredKickRewarnAfter +// passes. func logUndeliveredReviewKick(logger *slog.Logger, auditor reviewFixAuditor, k review.DispatchKick, err error) { if logger == nil { logger = slog.Default() } + if !undeliveredKicks.shouldReport(k, err, time.Now()) { + logger.Debug("review swarm kick still not delivered", + "kind", k.Kind, "agent", k.Agent, "pr", k.PRRef, "head", k.HeadSHA, "error", err) + return + } logger.Warn("review swarm kick not delivered; re-planned next cycle", "kind", k.Kind, "agent", k.Agent, "pr", k.PRRef, "head", k.HeadSHA, "error", err) if auditor == nil { @@ -112,6 +123,41 @@ func logUndeliveredReviewKick(logger *slog.Logger, auditor reviewFixAuditor, k r fmt.Sprintf("kind=%s pr=%s head=%s agent=%s error=%v", k.Kind, k.PRRef, k.HeadSHA, k.Agent, err), k.Agent) } +const ( + undeliveredKickRewarnAfter = 30 * time.Minute + undeliveredKickMaxTracked = 1024 +) + +type undeliveredKickReport struct { + cause string + at time.Time +} + +type undeliveredKickTracker struct { + mu sync.Mutex + seen map[string]undeliveredKickReport +} + +var undeliveredKicks = &undeliveredKickTracker{seen: map[string]undeliveredKickReport{}} + +func (t *undeliveredKickTracker) shouldReport(k review.DispatchKick, err error, now time.Time) bool { + key := k.Kind + "\x00" + k.Agent + "\x00" + k.PRRef + "\x00" + k.HeadSHA + cause := "" + if err != nil { + cause = err.Error() + } + t.mu.Lock() + defer t.mu.Unlock() + if prev, ok := t.seen[key]; ok && prev.cause == cause && now.Sub(prev.at) < undeliveredKickRewarnAfter { + return false + } + if len(t.seen) >= undeliveredKickMaxTracked { + t.seen = map[string]undeliveredKickReport{} + } + t.seen[key] = undeliveredKickReport{cause: cause, at: now} + return true +} + // governorKickDisplacementLimit is how many consecutive cycles an agent's // governor kick may be deferred behind a delivered fix kick before the // governor kick goes first for one cycle. Two: a single displacement is the diff --git a/src/cmd/hive/review_kick_order_test.go b/src/cmd/hive/review_kick_order_test.go index 6822ee190b..8a45547f40 100644 --- a/src/cmd/hive/review_kick_order_test.go +++ b/src/cmd/hive/review_kick_order_test.go @@ -87,6 +87,28 @@ func TestLogUndeliveredReviewKickNamesPR(t *testing.T) { logUndeliveredReviewKick(logger, nil, k, errors.New("x")) } +// The same undelivered kick (same PR head, same cause) repeats every cycle +// while the pane is not ready; only the first occurrence may warn/audit. +func TestLogUndeliveredReviewKickDedupesRepeats(t *testing.T) { + var buf bytes.Buffer + logger := slog.New(slog.NewTextHandler(&buf, nil)) + auditor := &recordingFixAuditor{} + k := review.DispatchKick{Kind: "fix", Agent: "dedupe-agent", PRRef: "hivecommons/hive#17724", HeadSHA: "abc1234"} + for i := 0; i < 5; i++ { + logUndeliveredReviewKick(logger, auditor, k, errors.New("pane not ready")) + } + if n := strings.Count(buf.String(), "review swarm kick not delivered"); n != 1 { + t.Fatalf("warn count = %d, want 1: %s", n, buf.String()) + } + if len(auditor.entries) != 1 { + t.Fatalf("audit entries = %d, want 1", len(auditor.entries)) + } + logUndeliveredReviewKick(logger, auditor, k, errors.New("different cause")) + if n := strings.Count(buf.String(), "review swarm kick not delivered"); n != 2 { + t.Fatalf("warn count after cause change = %d, want 2", n) + } +} + // hivecommons/hive#15166: a fix kick delivered first leaves the lane busy, so // the governor kick behind it is deferred; an adopted PR under // changes_requested is re-planned as a fix kick every cycle, so the scanner's diff --git a/src/cmd/hive/selfreport_mutation_hooks.go b/src/cmd/hive/selfreport_mutation_hooks.go index 48e7ac77d1..1d450607e4 100644 --- a/src/cmd/hive/selfreport_mutation_hooks.go +++ b/src/cmd/hive/selfreport_mutation_hooks.go @@ -12,7 +12,8 @@ import ( // hooks to the self-report tracker: a journaled external effect that stays // Unknown/Planned past mutation.DefaultStuckAfter while callers keep retrying // it is reported as fleetreport.ClassEffectLedgerOpStuck (key = the op's -// logical ID), and the condition clears when the op is applied or reconciled. +// logical ID), and the condition clears when the op is applied, reconciled or +// expired past its kind's unresolved TTL (Boundary.UnresolvedTTL). func installMutationSelfReportHooks(b *mutation.Boundary) { if b == nil { return diff --git a/src/cmd/hive/stage_engine.go b/src/cmd/hive/stage_engine.go index 6038b3da79..f856c9bfcb 100644 --- a/src/cmd/hive/stage_engine.go +++ b/src/cmd/hive/stage_engine.go @@ -39,6 +39,10 @@ func stageEngineOptions(cfg *config.Config, logger *slog.Logger) *stageengine.Op if logger == nil { logger = slog.Default() } + var wf *wfstore.Store + if cfg != nil { + wf = wfstore.Open(wfstore.Root(cfg.Data.WorkflowsDir)) + } stageEngine.opts = &stageengine.Options{ Mode: stageEngineConfiguredMode(cfg), Store: stagelabels.Shared(), @@ -46,10 +50,10 @@ func stageEngineOptions(cfg *config.Config, logger *slog.Logger) *stageengine.Op PR: stageengine.PRDeriver(), Logger: logger, Metrics: stageengine.DefaultCounters(), - WorkflowFor: stageEngineWorkflowFor(cfg, wfstore.Open(wfstore.Root(cfg.Data.WorkflowsDir)), logger), + WorkflowFor: stageEngineWorkflowFor(cfg, wf, logger), } if cfg != nil { - stageEngine.opts.WorkflowProvisioned = wfstore.Open(wfstore.Root(cfg.Data.WorkflowsDir)) + stageEngine.opts.WorkflowProvisioned = wf stageEngine.opts.WorkflowLabelGC = func() bool { return cfg.Workflow.LabelGCEnabled() } } }) diff --git a/src/cmd/hive/stage_engine_nilcfg_test.go b/src/cmd/hive/stage_engine_nilcfg_test.go new file mode 100644 index 0000000000..951b091a47 --- /dev/null +++ b/src/cmd/hive/stage_engine_nilcfg_test.go @@ -0,0 +1,32 @@ +package main + +import ( + "sync" + "testing" +) + +// A nil config must not panic while building the process-wide options +// (hivecommons/hive#17736); the sync.Once is reset so the test is the first caller. +func TestStageEngineOptions_NilConfigDoesNotPanic(t *testing.T) { + savedOpts := stageEngine.opts + stageEngine.once = sync.Once{} + stageEngine.opts = nil + t.Cleanup(func() { + stageEngine.once = sync.Once{} + stageEngine.opts = savedOpts + if savedOpts != nil { + stageEngine.once.Do(func() {}) + } + }) + + opts := stageEngineOptions(nil, nil) + if opts == nil { + t.Fatal("stageEngineOptions(nil, nil) returned nil Options") + } + if opts.WorkflowFor != nil { + t.Error("WorkflowFor should be nil without a config") + } + if opts.WorkflowProvisioned != nil { + t.Error("WorkflowProvisioned should be nil without a config") + } +} diff --git a/src/crux/agent/agent_working_test.go b/src/crux/agent/agent_working_test.go index ee005d9c2e..96c333ebc5 100644 --- a/src/crux/agent/agent_working_test.go +++ b/src/crux/agent/agent_working_test.go @@ -1,8 +1,11 @@ package agent import ( + "bytes" + "log/slog" "os" "path/filepath" + "strings" "testing" "time" @@ -99,6 +102,50 @@ func TestPaneShowsAgentWorking_OMPMidTurn(t *testing.T) { } } +// A prompt wait that times out because the agent is still mid-turn must not +// log the WARN pane dump: the slog feed files that line as agent_pane_timeout, +// so every re-kick that outwaited a long turn self-reported (#17993). A pane +// that never reaches a prompt for any other reason still dumps at WARN. +func TestWaitForInputPrompt_TimeoutWhileWorkingIsNotWarned(t *testing.T) { + origTimeout, origPoll := inputPromptTimeout, inputPromptPollInterval + inputPromptTimeout = 200 * time.Millisecond + inputPromptPollInterval = 10 * time.Millisecond + defer func() { + inputPromptTimeout, inputPromptPollInterval = origTimeout, origPoll + }() + + run := func(pane string) string { + t.Helper() + var logBuf bytes.Buffer + m := NewManager( + map[string]config.AgentConfig{"quality": {Backend: "claude"}}, + slog.New(slog.NewTextHandler(&logBuf, nil)), ProjectContext{}) + m.mu.Lock() + agent := m.agents["quality"] + m.mu.Unlock() + ft := termSeams(m) + ft.captureVisiblePane = func(*AgentProcess) string { return pane } + ft.capturePane = func(*AgentProcess) string { return pane } + if m.waitForInputPromptForAgent(agent) { + t.Fatalf("pane %q: returned ready; want not ready", pane) + } + return logBuf.String() + } + + busy := run(" ◉ Working · 42.6 KiB esc interrupt Claude Fable 5\n❯") + if strings.Contains(busy, "prompt timeout — dumping pane") { + t.Errorf("mid-turn timeout logged a WARN pane dump; want an INFO line only:\n%s", busy) + } + if !strings.Contains(busy, "prompt wait ended: agent still mid-turn") { + t.Errorf("mid-turn timeout did not log the mid-turn INFO line:\n%s", busy) + } + + stuck := run("some unrecognised screen with no prompt") + if !strings.Contains(stuck, "prompt timeout — dumping pane") { + t.Errorf("non-working timeout must still dump the pane at WARN:\n%s", stuck) + } +} + // Bob-Shell 2.x keeps the "❯" input box painted while a response streams and // signals busy only with "⠇ Processing… (Enter to steer, Tab to queue)". The // busy pane must read as working (delivery gate) and NOT as a finished turn diff --git a/src/crux/agent/git_identity_warn_dedupe.go b/src/crux/agent/git_identity_warn_dedupe.go new file mode 100644 index 0000000000..b3d04f9913 --- /dev/null +++ b/src/crux/agent/git_identity_warn_dedupe.go @@ -0,0 +1,37 @@ +package agent + +import ( + "crypto/sha256" + "os" + "sync" +) + +// strayGitIdentityWarned remembers, per (agent, config path), the content hash +// of the git config the stray-identity WARN was last emitted for. The launch +// check runs every cycle; without this the identical informational WARN would +// repeat and trip the repeated-error selfreport (hivecommons/hive#17703). +// State is in-memory, so a hive restart warns once more. +type strayGitIdentityWarned struct { + mu sync.Mutex + seen map[string][sha256.Size]byte +} + +// shouldWarn reports whether the finding at path is new for agentName: true on +// first sight and whenever the file content changed since the last warning. +func (s *strayGitIdentityWarned) shouldWarn(agentName, path string) bool { + var sum [sha256.Size]byte + if data, err := os.ReadFile(path); err == nil { + sum = sha256.Sum256(data) + } + key := agentName + "\x00" + path + s.mu.Lock() + defer s.mu.Unlock() + if prev, ok := s.seen[key]; ok && prev == sum { + return false + } + if s.seen == nil { + s.seen = make(map[string][sha256.Size]byte) + } + s.seen[key] = sum + return true +} diff --git a/src/crux/agent/interactive_home.go b/src/crux/agent/interactive_home.go index ee7ed1e9a1..964849fcb0 100644 --- a/src/crux/agent/interactive_home.go +++ b/src/crux/agent/interactive_home.go @@ -538,6 +538,9 @@ func (m *Manager) warnOnStrayGitIdentity(agentName, home string) { if !gitConfigDeclaresUserIdentity(path) { continue } + if !m.strayGitIdentity.shouldWarn(agentName, path) { + continue + } m.logger.Warn("agent git config declares a user identity outside the system /etc/gitconfig; commits still use the pinned per-lane GIT_AUTHOR_*/GIT_COMMITTER_* identity, but `git config user.email` reads this shared value instead (hivecommons/hive#9478)", "agent", agentName, "config", path) } diff --git a/src/crux/agent/interactive_home_gitconfig_test.go b/src/crux/agent/interactive_home_gitconfig_test.go index 6140b0c4cd..c2014e8cab 100644 --- a/src/crux/agent/interactive_home_gitconfig_test.go +++ b/src/crux/agent/interactive_home_gitconfig_test.go @@ -132,6 +132,40 @@ func TestWarnOnStrayGitIdentity(t *testing.T) { } } +// TestWarnOnStrayGitIdentity_OncePerAgentAndContent: the check runs every +// cycle, so an unchanged file must warn once, a changed file or another agent +// must warn again (hivecommons/hive#17703). +func TestWarnOnStrayGitIdentity_OncePerAgentAndContent(t *testing.T) { + const marker = "declares a user identity" + home := t.TempDir() + path := filepath.Join(home, ".gitconfig") + write := func(body string) { + t.Helper() + if err := os.WriteFile(path, []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + write("[user]\n\temail = a@example.com\n") + + m, logBuf := gitConfigHomeManager(t) + m.warnOnStrayGitIdentity("scanner", home) + m.warnOnStrayGitIdentity("scanner", home) + if got := strings.Count(logBuf.String(), marker); got != 1 { + t.Fatalf("repeat with unchanged config warned %d times, want 1: %s", got, logBuf.String()) + } + + write("[user]\n\temail = b@example.com\n") + m.warnOnStrayGitIdentity("scanner", home) + if got := strings.Count(logBuf.String(), marker); got != 2 { + t.Fatalf("changed config warned %d times total, want 2", got) + } + + m.warnOnStrayGitIdentity("reviewer", home) + if got := strings.Count(logBuf.String(), marker); got != 3 { + t.Fatalf("different agent warned %d times total, want 3", got) + } +} + func TestGitConfigDeclaresUserIdentity(t *testing.T) { dir := t.TempDir() cases := []struct { diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 87b7d9b7b8..52b6587b61 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -571,7 +571,8 @@ type Manager struct { // own mutex: the pane pollers that consult it must not take m.mu. rateLimitRelaunchMu sync.Mutex lastRateLimitRelaunch time.Time - cliMemory cliMemoryState // shared-store warning and drain hook, see cli_memory_drain.go + cliMemory cliMemoryState // shared-store warning and drain hook, see cli_memory_drain.go + strayGitIdentity strayGitIdentityWarned // warn-once state, see git_identity_warn_dedupe.go // thrashMu guards thrash — its own mutex, NEVER m.mu: the breaker runs on // the output-capture goroutines, and taking m.mu there risks startup @@ -809,6 +810,9 @@ type Manager struct { // sandboxOpenShellLauncherFactory is the same seam for // sandbox.runtime: openshell (ADR-0026). sandboxOpenShellLauncherFactory func(config.SandboxOpenShellConfig) sandbox.Launcher + // sandboxSubstrateLauncherFactory is the same seam for + // sandbox.runtime: substrate (ADR-0026 phase 2). + sandboxSubstrateLauncherFactory func(config.SandboxSubstrateConfig) sandbox.Launcher sandboxRunner sandboxCommandRunner // sandboxRunPersister overrides persistSandboxRun in tests. sandboxRunPersister func(workDir string, entry SandboxRunHistoryEntry) error @@ -1278,6 +1282,8 @@ func (m *Manager) waitForInputPromptForAgentReason(agent *AgentProcess, abort fu defer ticker.Stop() menuEscapes := 0 dialogAnswers := 0 + // lastWorking records whether the most recent poll saw the agent mid-turn. + lastWorking := false // Inference-routed agents run Claude Code too, so a permission dialog in // their pane is answered on the same terms as a plain claude agent's. dialogBackend := backend @@ -1291,6 +1297,18 @@ func (m *Manager) waitForInputPromptForAgentReason(agent *AgentProcess, abort fu } select { case <-deadline: + // An agent still mid-turn at the deadline is busy, not stuck: the + // kick paths already report it as busy and leave a genuinely stalled + // turn to the busy-no-activity watchdog (#12268). Dumping the pane at + // WARN here fed the agent_pane_timeout self-report every time a + // re-kick outwaited a long turn (#17993). + if lastWorking { + m.logger.Info("prompt wait ended: agent still mid-turn", + "agent", agent.Name, + "session", agent.tmuxSession, + "waited", inputPromptTimeout) + return false, kickPromptNotReadyReason + } m.logger.Warn("prompt timeout — dumping pane", "agent", agent.Name, "session", agent.tmuxSession) @@ -1317,6 +1335,7 @@ func (m *Manager) waitForInputPromptForAgentReason(agent *AgentProcess, abort fu // Check the visible pane only: a dismissed consent screen // lingers in the scrollback that captureTmuxPaneForAgent sees. visible := m.captureVisiblePaneForAgent(agent) + lastWorking = false if paneShowsCopilotQuestionForm(visible) { m.dismissCopilotQuestionFormForKick(agent) continue @@ -1349,6 +1368,7 @@ func (m *Manager) waitForInputPromptForAgentReason(agent *AgentProcess, abort fu // Use the visible pane only for the same reason as above: a // finished task's working marker lingers in scrollback. if paneShowsAgentWorking(visible) { + lastWorking = true continue } if paneShowsInputPrompt(visible) { @@ -1579,59 +1599,6 @@ func (m *Manager) RemoveAgent(name string) { )) } -// AuthorizeReviewRequest enforces the policy for the review-request watcher. -// It mirrors AuthorizePROpen's forge-resistance, but grants on capability as -// well as mode (hivecommons/hive#7485). -// -// The review-request watcher is the SANCTIONED path: its doc comment tells an -// agent to write a request file "INSTEAD of running `gh pr review` from its -// own shell", because the file relay is App-authored and lands on the audit -// trail. But it was gated with AuthorizePROpen, which requires CanPush() — -// while the proxy, which is what a direct `gh pr review` goes through, grants -// the same write on Converse alone: -// -// MinMode: agent.ModeIssuesAndPRs, Capability: agent.AgentCapabilities.CanConverse -// — pkg/proxy/rules.go:152 -// -// So an ADVISORY+converse agent could post a review by going AROUND the relay -// but not THROUGH it: the audited path was strictly stricter than the -// unaudited one, which is exactly backwards — it pressures agents off the -// trail. This aligns the relay with the proxy so the sanctioned route is never -// the more restricted one. Converse stays conversation-only: it does not grant -// pushing, opening PRs, or merging, which all remain on the mode ladder. -// -// A nil manager or unknown agent is denied. -func (m *Manager) AuthorizeReviewRequest(agentName string, fileUID int) error { - if strings.TrimSpace(agentName) == "" { - return fmt.Errorf("no agent named in the request") - } - // Forge check: when we have a UID map and a real owning UID, the file owner - // must BE this agent. - if m.uidMap != nil && fileUID > 0 { - owner := m.uidMap.LookupByUID(fileUID) - if owner == "" { - return fmt.Errorf("request file owned by unknown uid %d (not a registered agent)", fileUID) - } - if owner != agentName { - return fmt.Errorf("request claims agent %q but file is owned by agent %q (uid %d)", agentName, owner, fileUID) - } - } - m.mu.RLock() - agent := m.agents[agentName] - m.mu.RUnlock() - if agent == nil { - return fmt.Errorf("unknown agent %q", agentName) - } - if m.agentCapabilities(agent).CanConverse() { - return nil - } - if !m.agentMode(agent).CanPush() { - return fmt.Errorf("agent %q may not review PRs at this ACMM level (mode %s) and does not have the `converse` capability", - agentName, m.agentMode(agent).String()) - } - return nil -} - func (m *Manager) bobPreflightUnlocked(agent *AgentProcess) { if agent == nil { return diff --git a/src/crux/agent/manager_mint.go b/src/crux/agent/manager_mint.go new file mode 100644 index 0000000000..fe8896f327 --- /dev/null +++ b/src/crux/agent/manager_mint.go @@ -0,0 +1,98 @@ +package agent + +import ( + "context" + "errors" + "net" + "net/url" + + "github.com/hivecommons/hive/crux" +) + +// mintAgentTokenUnlocked mints the per-agent tier-scoped GitHub App token +// (WriteAgentToken) and the opt-in short-lived mint token for the agent. It is +// the single mint step every (re)launch path — Start, Resume, Restart, +// RestartWithBootstrap — runs before launchInTmux (#3962: previously only +// Start minted, so an agent relaunched via Resume/Restart never got a token +// until the whole process restarted; its cache file stayed 0 bytes and gh/git +// push failed silently). +// +// MUST be called with m.mu RELEASED. WriteAgentToken makes an outbound GitHub +// API call (through the MITM egress proxy, which the production incident +// showed can hang), and issueAgentMintToken can do the same. Holding m.mu +// across a hang stalls AllStatuses()/the heartbeat for the WHOLE fleet — the +// exact liveness flap Start's phase split was built to avoid. The function +// reads only immutable agent identity (Name, UID) and Config via agentMode, +// and writes no m.mu-guarded AgentProcess field, so running it lock-free is +// race-free. m.appAuth is read without the lock exactly as Start's unlocked +// phase always did: it is injected once at wiring time, before agents start. +func (m *Manager) mintAgentTokenUnlocked(ctx context.Context, agent *AgentProcess) { + if m.appAuth == nil || !m.agentTokenCacheEnabled(agent) { + return + } + tier := m.agentTokenTier(agent) + if err := m.appAuth.WriteAgentToken(ctx, agent.Name, tier, agent.UID); err != nil && errors.Is(err, crux.ErrGitHubAppNotInstalled) { + // Static misconfiguration: every tick would fail identically, so log + // once per agent until a mint succeeds instead of every supervisor tick. + if _, seen := m.appNotInstalledWarned.LoadOrStore(agent.Name, struct{}{}); !seen { + m.logger.Warn("github app installation id is unset — per-agent scoped token NOT delivered, this agent's GitHub writes (gh + git push) will FAIL; set github.installation_id or install the App on the hive's org so discovery resolves it", + "agent", agent.Name, "tier", tier, "error", err) + } + m.observeCondition(crux.ClassGitHubAppNotInstalled, agent.Name, "github app installation id is unset: "+err.Error(), "") + } else if err != nil && isForgeTransportErr(err) { + // The GitHub host did not answer (dial timeout, connection refused, + // deadline exceeded). Nothing about the App key or installation can + // be concluded from that, so report it as a forge-reachability + // condition rather than a mint failure — the latter tells the + // operator to re-check App permissions, which is wrong during an + // egress outage. The mint-failed condition is cleared so a hive that + // slid from a real mint error into an outage does not hold both open. + m.logger.Warn("per-agent scoped token NOT delivered — GitHub host unreachable from the pod; this agent's GitHub writes (gh + git push) will FAIL until egress is restored", + "agent", agent.Name, "tier", tier, "error", err) + m.clearCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name) + m.observeCondition(crux.ClassForgeUnreachable, agent.Name, "github host unreachable during token mint: "+err.Error(), "") + } else if err != nil { + // Be precise about the blast radius. Since audit H3 the shared-cache + // fallback is GONE: gh-wrapper.sh and git-credential-hive.sh no + // longer fall back to /var/run/hive-metrics/gh-app-token.cache (the + // FULL installation token, now owner-only 0600). They FAIL LOUD when + // the per-agent scoped token is absent rather than silently + // escalating this agent to full privilege. So a delivery failure here + // means this agent's GitHub writes (gh + git push) will fail until + // token delivery is repaired — not a silent privilege escalation. + m.logger.Warn("per-agent scoped token NOT delivered — this agent's GitHub writes (gh + git push) will FAIL (no shared-cache fallback by design; see audit H3)", + "agent", agent.Name, "tier", tier, "error", err) + m.clearCondition(crux.ClassForgeUnreachable, agent.Name) + m.observeCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name, "github app token mint failed: "+err.Error(), "") + } else { + m.appNotInstalledWarned.Delete(agent.Name) + m.clearCondition(crux.ClassGitHubAppNotInstalled, agent.Name) + m.clearCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name) + m.clearCondition(crux.ClassForgeUnreachable, agent.Name) + } + // Additionally issue an opt-in short-lived mint token (no-op when the mint + // is disabled). This is additive and fail-safe — it never blocks launch. + if agent.UID > 0 { + m.issueAgentMintToken(agent.Name, tier, agent.UID) + } +} + +// isForgeTransportErr reports whether a mint error never got an answer from +// the GitHub host: a deadline, a *url.Error (dial/TLS/timeout from the HTTP +// client) or any net.Error. An HTTP status from GitHub (401/404/422) is not +// a transport error and keeps its mint-failed classification. Mirrors the +// dashboard's forge probe (#17427) so both report the same outage. +func isForgeTransportErr(err error) bool { + if err == nil { + return false + } + if errors.Is(err, context.DeadlineExceeded) { + return true + } + var urlErr *url.Error + if errors.As(err, &urlErr) { + return true + } + var netErr net.Error + return errors.As(err, &netErr) +} diff --git a/src/crux/agent/manager_review_authz.go b/src/crux/agent/manager_review_authz.go new file mode 100644 index 0000000000..22c715a975 --- /dev/null +++ b/src/crux/agent/manager_review_authz.go @@ -0,0 +1,59 @@ +package agent + +import ( + "fmt" + "strings" +) + +// AuthorizeReviewRequest enforces the policy for the review-request watcher. +// It mirrors AuthorizePROpen's forge-resistance, but grants on capability as +// well as mode (hivecommons/hive#7485). +// +// The review-request watcher is the SANCTIONED path: its doc comment tells an +// agent to write a request file "INSTEAD of running `gh pr review` from its +// own shell", because the file relay is App-authored and lands on the audit +// trail. But it was gated with AuthorizePROpen, which requires CanPush() — +// while the proxy, which is what a direct `gh pr review` goes through, grants +// the same write on Converse alone: +// +// MinMode: agent.ModeIssuesAndPRs, Capability: agent.AgentCapabilities.CanConverse +// — pkg/proxy/rules.go:152 +// +// So an ADVISORY+converse agent could post a review by going AROUND the relay +// but not THROUGH it: the audited path was strictly stricter than the +// unaudited one, which is exactly backwards — it pressures agents off the +// trail. This aligns the relay with the proxy so the sanctioned route is never +// the more restricted one. Converse stays conversation-only: it does not grant +// pushing, opening PRs, or merging, which all remain on the mode ladder. +// +// A nil manager or unknown agent is denied. +func (m *Manager) AuthorizeReviewRequest(agentName string, fileUID int) error { + if strings.TrimSpace(agentName) == "" { + return fmt.Errorf("no agent named in the request") + } + // Forge check: when we have a UID map and a real owning UID, the file owner + // must BE this agent. + if m.uidMap != nil && fileUID > 0 { + owner := m.uidMap.LookupByUID(fileUID) + if owner == "" { + return fmt.Errorf("request file owned by unknown uid %d (not a registered agent)", fileUID) + } + if owner != agentName { + return fmt.Errorf("request claims agent %q but file is owned by agent %q (uid %d)", agentName, owner, fileUID) + } + } + m.mu.RLock() + agent := m.agents[agentName] + m.mu.RUnlock() + if agent == nil { + return fmt.Errorf("unknown agent %q", agentName) + } + if m.agentCapabilities(agent).CanConverse() { + return nil + } + if !m.agentMode(agent).CanPush() { + return fmt.Errorf("agent %q may not review PRs at this ACMM level (mode %s) and does not have the `converse` capability", + agentName, m.agentMode(agent).String()) + } + return nil +} diff --git a/src/crux/agent/manager_sandbox.go b/src/crux/agent/manager_sandbox.go index 244776a90d..d73bc3e287 100644 --- a/src/crux/agent/manager_sandbox.go +++ b/src/crux/agent/manager_sandbox.go @@ -19,6 +19,7 @@ import ( func init() { config.SetOpenShellPreflight(openShellPreflightWarnings) + config.SetSubstratePreflight(substratePreflightWarnings) } // openShellPreflightWarnings checks each distinct gateway the openshell @@ -113,7 +114,7 @@ func (m *Manager) validateSandboxKickLocked(agent *AgentProcess) (string, error) // does, instead of failing it after the state machine has moved. runtime := agent.Config.SandboxRuntime(m.sandboxConfig) if !config.ValidSandboxRuntimes[runtime] { - return "", fmt.Errorf("agent %s sandbox runtime %q is not one hive has (%s, %s or %s)", agent.Name, runtime, config.SandboxRuntimePodman, config.SandboxRuntimeJob, config.SandboxRuntimeOpenShell) + return "", fmt.Errorf("agent %s sandbox runtime %q is not one hive has (%s, %s, %s or %s)", agent.Name, runtime, config.SandboxRuntimePodman, config.SandboxRuntimeJob, config.SandboxRuntimeOpenShell, config.SandboxRuntimeSubstrate) } switch runtime { case config.SandboxRuntimeJob: @@ -130,6 +131,15 @@ func (m *Manager) validateSandboxKickLocked(agent *AgentProcess) (string, error) if effectiveBackend(agent) == "cline" { return "", fmt.Errorf("agent %s: the Cline backend is not supported on the openshell sandbox runtime (no credential path into the sandbox); use podman or job", agent.Name) } + case config.SandboxRuntimeSubstrate: + // ADR-0026 phase 2: same contract as openshell — inert unless + // selected, refused before the state machine moves. + if err := SubstrateOptionsFromConfig(agent.Config.SandboxSubstrate(m.sandboxConfig)).Validate(); err != nil { + return "", fmt.Errorf("agent %s sandbox substrate runtime: %w", agent.Name, err) + } + if effectiveBackend(agent) == "cline" { + return "", fmt.Errorf("agent %s: the Cline backend is not supported on the substrate sandbox runtime (no credential path into the sandbox); use podman or job", agent.Name) + } } if effectiveBackend(agent) == "cline" { if err := validateClineSandbox(agent, runtime, agent.Config.SandboxJob(m.sandboxConfig), m.project.TaskMCPURL); err != nil { @@ -151,6 +161,8 @@ func (m *Manager) startSandboxKickLocked(agent *AgentProcess, message, source st launcher = m.jobLauncherLocked(agent.Config.SandboxJob(m.sandboxConfig)) case config.SandboxRuntimeOpenShell: launcher = m.openshellLauncherLocked(agent.Config.SandboxOpenShell(m.sandboxConfig)) + case config.SandboxRuntimeSubstrate: + launcher = m.substrateLauncherLocked(agent.Config.SandboxSubstrate(m.sandboxConfig)) } now := time.Now() agent.State = StateRunning diff --git a/src/crux/agent/manager_sandbox_substrate.go b/src/crux/agent/manager_sandbox_substrate.go new file mode 100644 index 0000000000..861ec4936c --- /dev/null +++ b/src/crux/agent/manager_sandbox_substrate.go @@ -0,0 +1,80 @@ +package agent + +import ( + "context" + "os" + "strings" + "time" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/sandbox" +) + +// sandbox.runtime: substrate wiring (ADR-0026 phase 2); the sibling of the +// openshell helpers in manager_sandbox.go. + +// substratePreflightWarnings checks each distinct control plane the +// substrate agents resolve to, once. +func substratePreflightWarnings(blocks []config.SandboxSubstrateConfig) []string { + var out []string + seen := map[string]bool{} + for _, b := range blocks { + opts := SubstrateOptionsFromConfig(b) + key := opts.Endpoint + "\x00" + opts.Atespace + "\x00" + opts.Token + if seen[key] { + continue + } + seen[key] = true + out = append(out, sandbox.DefaultSubstratePreflighter.Check(context.Background(), opts)...) + } + return out +} + +func (m *Manager) setSandboxSubstrateLauncherFactoryForTest(f func(config.SandboxSubstrateConfig) sandbox.Launcher) { + m.mu.Lock() + defer m.mu.Unlock() + m.sandboxSubstrateLauncherFactory = f +} + +// substrateLauncherLocked returns the launcher for a sandbox.runtime: +// substrate kick: the injected factory's, or a real Control-API-backed +// sandbox.SubstrateLauncher. +func (m *Manager) substrateLauncherLocked(cfg config.SandboxSubstrateConfig) sandbox.Launcher { + if m.sandboxSubstrateLauncherFactory != nil { + return m.sandboxSubstrateLauncherFactory(cfg) + } + opts := SubstrateOptionsFromConfig(cfg) + opts.Logger = m.logger + return &sandbox.SubstrateLauncher{Options: opts} +} + +// SubstrateOptionsFromConfig converts the operator's substrate block into +// the launcher's options. A token named by token_env is read from the +// environment so it never sits in hive.yaml; otherwise the launcher reads +// the projected token file per request. +func SubstrateOptionsFromConfig(cfg config.SandboxSubstrateConfig) sandbox.SubstrateOptions { + opts := sandbox.SubstrateOptions{ + Endpoint: cfg.Endpoint, + Atespace: cfg.Atespace, + TokenFile: cfg.TokenFile, + CAFile: cfg.CAFile, + InsecureTLS: cfg.InsecureTLS, + Timeout: time.Duration(cfg.TimeoutS) * time.Second, + RouterURL: cfg.RouterURL, + RouterClientBundle: cfg.RouterClientBundle, + RouterCAFile: cfg.RouterCAFile, + BridgePort: cfg.BridgePort, + Image: cfg.Image, + Command: append([]string(nil), cfg.BridgeCommand...), + CPUMillicores: cfg.CPUMillicores, + MemoryBytes: cfg.MemoryBytes, + WorkerSelector: cfg.WorkerSelector, + SnapshotOnIdle: cfg.SnapshotOnIdle, + AllowHosts: append([]string(nil), cfg.AllowHosts...), + PolicyProfile: cfg.PolicyProfile, + } + if env := strings.TrimSpace(cfg.TokenEnv); env != "" { + opts.Token = os.Getenv(env) + } + return opts +} diff --git a/src/crux/agent/manager_tokens.go b/src/crux/agent/manager_tokens.go index e5694a82ba..fcdb70ebe7 100644 --- a/src/crux/agent/manager_tokens.go +++ b/src/crux/agent/manager_tokens.go @@ -2,7 +2,6 @@ package agent import ( "context" - "errors" "fmt" "os" "strings" @@ -811,60 +810,6 @@ func (m *Manager) refreshAgentTokens(ctx context.Context) { func truncateStr(s string, maxRunes int) string { return textutil.Head(s, maxRunes) } -// mintAgentTokenUnlocked mints the per-agent tier-scoped GitHub App token -// (WriteAgentToken) and the opt-in short-lived mint token for the agent. It is -// the single mint step every (re)launch path — Start, Resume, Restart, -// RestartWithBootstrap — runs before launchInTmux (#3962: previously only -// Start minted, so an agent relaunched via Resume/Restart never got a token -// until the whole process restarted; its cache file stayed 0 bytes and gh/git -// push failed silently). -// -// MUST be called with m.mu RELEASED. WriteAgentToken makes an outbound GitHub -// API call (through the MITM egress proxy, which the production incident -// showed can hang), and issueAgentMintToken can do the same. Holding m.mu -// across a hang stalls AllStatuses()/the heartbeat for the WHOLE fleet — the -// exact liveness flap Start's phase split was built to avoid. The function -// reads only immutable agent identity (Name, UID) and Config via agentMode, -// and writes no m.mu-guarded AgentProcess field, so running it lock-free is -// race-free. m.appAuth is read without the lock exactly as Start's unlocked -// phase always did: it is injected once at wiring time, before agents start. -func (m *Manager) mintAgentTokenUnlocked(ctx context.Context, agent *AgentProcess) { - if m.appAuth == nil || !m.agentTokenCacheEnabled(agent) { - return - } - tier := m.agentTokenTier(agent) - if err := m.appAuth.WriteAgentToken(ctx, agent.Name, tier, agent.UID); err != nil && errors.Is(err, crux.ErrGitHubAppNotInstalled) { - // Static misconfiguration: every tick would fail identically, so log - // once per agent until a mint succeeds instead of every supervisor tick. - if _, seen := m.appNotInstalledWarned.LoadOrStore(agent.Name, struct{}{}); !seen { - m.logger.Warn("github app installation id is unset — per-agent scoped token NOT delivered, this agent's GitHub writes (gh + git push) will FAIL; set github.installation_id or install the App on the hive's org so discovery resolves it", - "agent", agent.Name, "tier", tier, "error", err) - } - m.observeCondition(crux.ClassGitHubAppNotInstalled, agent.Name, "github app installation id is unset: "+err.Error(), "") - } else if err != nil { - // Be precise about the blast radius. Since audit H3 the shared-cache - // fallback is GONE: gh-wrapper.sh and git-credential-hive.sh no - // longer fall back to /var/run/hive-metrics/gh-app-token.cache (the - // FULL installation token, now owner-only 0600). They FAIL LOUD when - // the per-agent scoped token is absent rather than silently - // escalating this agent to full privilege. So a delivery failure here - // means this agent's GitHub writes (gh + git push) will fail until - // token delivery is repaired — not a silent privilege escalation. - m.logger.Warn("per-agent scoped token NOT delivered — this agent's GitHub writes (gh + git push) will FAIL (no shared-cache fallback by design; see audit H3)", - "agent", agent.Name, "tier", tier, "error", err) - m.observeCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name, "github app token mint failed: "+err.Error(), "") - } else { - m.appNotInstalledWarned.Delete(agent.Name) - m.clearCondition(crux.ClassGitHubAppNotInstalled, agent.Name) - m.clearCondition(crux.ClassGitHubAppTokenMintFailed, agent.Name) - } - // Additionally issue an opt-in short-lived mint token (no-op when the mint - // is disabled). This is additive and fail-safe — it never blocks launch. - if agent.UID > 0 { - m.issueAgentMintToken(agent.Name, tier, agent.UID) - } -} - // agentTokenCacheEnabled keeps the shared-UID read path opt-in and App-only. func (m *Manager) agentTokenCacheEnabled(a *AgentProcess) bool { return a.UID > 0 || (a.UID == 0 && m.project.SharedUIDReadToken && m.appAuth != nil) diff --git a/src/crux/agent/mint_forge_unreachable_test.go b/src/crux/agent/mint_forge_unreachable_test.go new file mode 100644 index 0000000000..69e8cdd055 --- /dev/null +++ b/src/crux/agent/mint_forge_unreachable_test.go @@ -0,0 +1,93 @@ +package agent + +import ( + "context" + "errors" + "net" + "net/url" + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/config" +) + +// failingMinter returns the configured error from WriteAgentToken. +type failingMinter struct{ err error } + +func (f *failingMinter) WriteAgentToken(context.Context, string, string, int) error { return f.err } + +type fakeNetErr struct{} + +func (fakeNetErr) Error() string { return "dial tcp 129.34.104.21:443: connect: connection refused" } +func (fakeNetErr) Timeout() bool { return false } +func (fakeNetErr) Temporary() bool { return false } + +var _ net.Error = fakeNetErr{} + +func mintClassManager(t *testing.T, minter AppTokenMinter) (*Manager, *AgentProcess) { + t.Helper() + t.Setenv("HIVE_WORK_DIR", t.TempDir()) + m := NewManager(map[string]config.AgentConfig{ + "worker": makeAgentConfig("claude", "sonnet"), + }, discardLogger(), ProjectContext{}) + m.mu.Lock() + a := m.agents["worker"] + a.UID = 2101 + m.mu.Unlock() + m.SetAppAuth(minter) + return m, a +} + +// A mint that never reaches the forge is a forge_unreachable condition, not a +// github_app_token_mint_failed one — the App is not implicated by an egress +// outage, and the mint-failed condition is cleared so both are never open. +func TestMintTransportErrorReportsForgeUnreachable(t *testing.T) { + cases := map[string]error{ + "deadline": context.DeadlineExceeded, + "url_error": &url.Error{Op: "Post", URL: "https://github.ibm.com/api/v3/app/installations/45218/access_tokens", Err: context.DeadlineExceeded}, + "net_error": errors.Join(errors.New("creating installation token"), fakeNetErr{}), + } + for name, err := range cases { + t.Run(name, func(t *testing.T) { + m, a := mintClassManager(t, &failingMinter{err: err}) + rec := recordConditions(t, m) + m.mintAgentTokenUnlocked(context.Background(), a) + if got := rec.active(crux.ClassForgeUnreachable); len(got) != 1 || got[0].Key != "worker" { + t.Fatalf("forge_unreachable active = %+v, want one for worker", got) + } + if got := rec.observations(crux.ClassGitHubAppTokenMintFailed); len(got) != 0 { + t.Fatalf("github_app_token_mint_failed observed on a transport error: %+v", got) + } + if got := rec.clears(crux.ClassGitHubAppTokenMintFailed); len(got) != 1 { + t.Fatalf("mint-failed clears = %d, want 1", len(got)) + } + }) + } +} + +// An answer from the forge that refuses the mint keeps the mint-failed class +// and clears any forge_unreachable left over from an outage. +func TestMintNonTransportErrorKeepsMintFailed(t *testing.T) { + m, a := mintClassManager(t, &failingMinter{err: errors.New("creating installation token: POST .../access_tokens: 401 Bad credentials")}) + rec := recordConditions(t, m) + m.mintAgentTokenUnlocked(context.Background(), a) + if got := rec.active(crux.ClassGitHubAppTokenMintFailed); len(got) != 1 { + t.Fatalf("github_app_token_mint_failed active = %+v, want one", got) + } + if got := rec.observations(crux.ClassForgeUnreachable); len(got) != 0 { + t.Fatalf("forge_unreachable observed on an HTTP error: %+v", got) + } + if got := rec.clears(crux.ClassForgeUnreachable); len(got) != 1 { + t.Fatalf("forge_unreachable clears = %d, want 1", len(got)) + } +} + +// A successful mint clears every mint-related condition, including the new one. +func TestMintSuccessClearsForgeUnreachable(t *testing.T) { + m, a := mintClassManager(t, &failingMinter{err: nil}) + rec := recordConditions(t, m) + m.mintAgentTokenUnlocked(context.Background(), a) + if got := rec.clears(crux.ClassForgeUnreachable); len(got) != 1 { + t.Fatalf("forge_unreachable clears = %d, want 1", len(got)) + } +} diff --git a/src/crux/agent/pluk_dirs.go b/src/crux/agent/pluk_dirs.go index 9a53f13836..9cac490c00 100644 --- a/src/crux/agent/pluk_dirs.go +++ b/src/crux/agent/pluk_dirs.go @@ -7,6 +7,9 @@ import ( "os" "path/filepath" "sync" + "syscall" + + "github.com/hivecommons/hive/crux/sharedfile" ) // plukRunDir is where pluk's run state and per-session logs live. @@ -46,32 +49,79 @@ func plukSessionLogPath(runDir, session string) string { // ensurePlukLogFile creates (or leaves alone) the session's JSONL log with a // mode peer agents can read, and returns its path. Both the O_CREATE mode and a -// following Chmod are needed: O_CREATE is umask-filtered, and an existing file +// following chmod are needed: O_CREATE is umask-filtered, and an existing file // from an earlier run under a tighter umask has to be widened back to the shared // mode rather than inherited as-is. +// +// The logs dir is group-writable by every agent uid (0770+setgid, no sticky +// bit), so anything found at path may have been planted by an agent: the open +// never follows a symlink, the descriptor is checked to be a regular file, +// and the owner and mode are applied through that descriptor, never by path. +// A path-based chmod/chown here would let one agent widen any dev-owned file +// to the shared group by linking its own session log at it (#17966). func ensurePlukLogFile(runDir, session string) (string, error) { path := plukSessionLogPath(runDir, session) - f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, plukLogFileMode) + f, err := openPlukLogNoFollow(path) + if errors.Is(err, fs.ErrPermission) || errors.Is(err, sharedfile.ErrNotRegular) { + // A log left behind by another uid (a pane shell's `>>` under a tight + // umask, or an earlier run as a different user) cannot be opened or + // chmodded by us, and a symlink or other non-regular entry at the path + // is not a log at all. The shared setgid logs dir is group-writable, so + // the entry can be replaced. Replacing beats leaving the session's log + // unreadable to peers, or following a planted link. + if rmErr := os.Remove(path); rmErr == nil { + f, err = openPlukLogNoFollow(path) + } + } if err != nil { return "", fmt.Errorf("creating %s: %w", path, err) } - if err := f.Close(); err != nil { - return "", fmt.Errorf("closing %s: %w", path, err) - } + defer f.Close() if os.Geteuid() == 0 { - if err := os.Chown(path, DevUID, NodeGID); err != nil { + if err := f.Chown(DevUID, NodeGID); err != nil { return "", fmt.Errorf("chowning %s to %d:%d: %w", path, DevUID, NodeGID, err) } } - if err := os.Chmod(path, plukLogFileMode); err != nil { + if err := f.Chmod(plukLogFileMode); err != nil { return "", fmt.Errorf("chmoding %s to %s: %w", path, plukLogFileMode, err) } + if err := f.Close(); err != nil { + return "", fmt.Errorf("closing %s: %w", path, err) + } return path, nil } +// openPlukLogNoFollow opens (creating if absent) the regular file at path for +// append without following a symlink there. A symlink is reported as +// sharedfile.ErrNotRegular (the kernel answers ELOOP), as is any other +// non-regular file found behind the descriptor. O_NONBLOCK keeps a planted +// FIFO from parking the launch path on open (it fails with ENXIO instead); +// the kernel ignores it for a regular file. +func openPlukLogNoFollow(path string) (*os.File, error) { + f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|syscall.O_NOFOLLOW|syscall.O_NONBLOCK, plukLogFileMode) + if err != nil { + // ELOOP: a symlink. ENXIO: a FIFO with no reader (O_NONBLOCK). Neither + // is a log; both are reported as not-regular so the caller replaces them. + if errors.Is(err, syscall.ELOOP) || errors.Is(err, syscall.ENXIO) { + return nil, fmt.Errorf("%w: %s (%v)", sharedfile.ErrNotRegular, path, err) + } + return nil, err + } + info, err := f.Stat() + if err != nil { + f.Close() + return nil, err + } + if !info.Mode().IsRegular() { + f.Close() + return nil, fmt.Errorf("%w: %s is %s", sharedfile.ErrNotRegular, path, info.Mode().Type()) + } + return f, nil +} + func ensurePlukRunDirs(runDir string) error { for _, name := range []string{"logs", "commands"} { dir := filepath.Join(runDir, name) diff --git a/src/crux/agent/pluk_dirs_branches_test.go b/src/crux/agent/pluk_dirs_branches_test.go index 45e6c39179..0d0affa096 100644 --- a/src/crux/agent/pluk_dirs_branches_test.go +++ b/src/crux/agent/pluk_dirs_branches_test.go @@ -6,7 +6,9 @@ import ( "os" "path/filepath" "strings" + "syscall" "testing" + "time" ) // pluk_dirs_branches_test.go covers the pluk_dirs.go branches the setgid-mode @@ -83,6 +85,42 @@ func TestEnsurePlukLogFileWidensExistingTightLog(t *testing.T) { } } +// TestEnsurePlukLogFileReplacesUnopenableLog pins the supervisor failure: a log +// left by another uid that we cannot open (modelled as mode 0000) is replaced +// in the group-writable logs dir instead of failing with permission denied. +func TestEnsurePlukLogFileReplacesUnopenableLog(t *testing.T) { + if os.Geteuid() == 0 { + t.Skip("root bypasses file permission checks") + } + runDir := filepath.Join(t.TempDir(), "pluk") + if err := ensurePlukRunDirs(runDir); err != nil { + t.Fatalf("ensure pluk dirs: %v", err) + } + + path := plukSessionLogPath(runDir, "hive-supervisor") + if err := os.WriteFile(path, nil, 0o600); err != nil { + t.Fatalf("seed log: %v", err) + } + if err := os.Chmod(path, 0); err != nil { + t.Fatalf("lock log: %v", err) + } + + got, err := ensurePlukLogFile(runDir, "hive-supervisor") + if err != nil { + t.Fatalf("ensure pluk log file with unopenable log: %v", err) + } + if got != path { + t.Errorf("returned path = %q, want %q", got, path) + } + info, err := os.Stat(path) + if err != nil { + t.Fatalf("stat %s: %v", path, err) + } + if perm := info.Mode().Perm(); perm != 0o660 { + t.Errorf("replaced log mode = %04o, want 0660", perm) + } +} + // TestEnsurePlukLogFileErrorWhenLogsDirMissing covers the OpenFile error // return: with no logs/ directory the create fails, the error is wrapped // (fs.ErrNotExist stays reachable through errors.Is), and it names the path. @@ -156,3 +194,89 @@ func TestEnsurePlukRunDirsRewidensTightenedDirs(t *testing.T) { } } } + +// TestEnsurePlukLogFileDoesNotFollowPlantedSymlink pins #17966: the logs dir is +// group-writable by every agent uid, so a symlink at the session log path may +// have been planted by an agent. The chmod (and, as root, the chown) must never +// reach the link's target; the link is replaced by a regular log instead. +func TestEnsurePlukLogFileDoesNotFollowPlantedSymlink(t *testing.T) { + runDir := filepath.Join(t.TempDir(), "pluk") + if err := ensurePlukRunDirs(runDir); err != nil { + t.Fatalf("ensure pluk dirs: %v", err) + } + secret := filepath.Join(t.TempDir(), "proxy-ca-key.pem") + if err := os.WriteFile(secret, []byte("private"), 0o600); err != nil { + t.Fatalf("seed secret: %v", err) + } + path := plukSessionLogPath(runDir, "hive-attacker") + if err := os.Symlink(secret, path); err != nil { + t.Fatalf("plant symlink: %v", err) + } + + got, err := ensurePlukLogFile(runDir, "hive-attacker") + if err != nil { + t.Fatalf("ensure pluk log file over a planted symlink: %v", err) + } + if got != path { + t.Errorf("returned path = %q, want %q", got, path) + } + + info, err := os.Stat(secret) + if err != nil { + t.Fatalf("stat secret: %v", err) + } + if perm := info.Mode().Perm(); perm != 0o600 { + t.Fatalf("symlink target mode = %04o, want 0600 untouched (chmod followed the planted link)", perm) + } + data, err := os.ReadFile(secret) + if err != nil || string(data) != "private" { + t.Fatalf("symlink target changed: %q, %v", data, err) + } + + li, err := os.Lstat(path) + if err != nil { + t.Fatalf("lstat %s: %v", path, err) + } + if !li.Mode().IsRegular() { + t.Fatalf("log path is %s, want a regular file replacing the planted link", li.Mode().Type()) + } + if perm := li.Mode().Perm(); perm != 0o660 { + t.Errorf("replaced log mode = %04o, want 0660", perm) + } +} + +// TestEnsurePlukLogFileReplacesPlantedFIFO covers the other non-regular entry +// an agent can create in the shared dir: a FIFO with no reader must neither +// block the launch path on open nor be kept; it is replaced like a symlink. +func TestEnsurePlukLogFileReplacesPlantedFIFO(t *testing.T) { + runDir := filepath.Join(t.TempDir(), "pluk") + if err := ensurePlukRunDirs(runDir); err != nil { + t.Fatalf("ensure pluk dirs: %v", err) + } + path := plukSessionLogPath(runDir, "hive-fifo") + if err := syscall.Mkfifo(path, 0o660); err != nil { + t.Fatalf("plant fifo: %v", err) + } + + done := make(chan error, 1) + go func() { + _, err := ensurePlukLogFile(runDir, "hive-fifo") + done <- err + }() + select { + case err := <-done: + if err != nil { + t.Fatalf("ensure pluk log file over a planted fifo: %v", err) + } + case <-time.After(5 * time.Second): + t.Fatal("ensure pluk log file blocked on a planted fifo") + } + + li, err := os.Lstat(path) + if err != nil { + t.Fatalf("lstat %s: %v", path, err) + } + if !li.Mode().IsRegular() { + t.Fatalf("log path is %s, want a regular file replacing the fifo", li.Mode().Type()) + } +} diff --git a/src/crux/agent/sandbox_substrate_runtime_test.go b/src/crux/agent/sandbox_substrate_runtime_test.go new file mode 100644 index 0000000000..dfd75d04fa --- /dev/null +++ b/src/crux/agent/sandbox_substrate_runtime_test.go @@ -0,0 +1,141 @@ +package agent + +import ( + "context" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/sandbox" +) + +// The Substrate runtime (ADR-0026 phase 2) routes a sandboxed kick through +// the launcher built from the agent's substrate block, refuses a kick whose +// options are invalid before the state machine moves, and stays inert for +// every agent that did not select it. + +const substrateTestImage = "ghcr.io/hivecommons/sandbox@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func substrateRuntimeManager(t *testing.T, override *config.AgentSandboxOverride, global config.AgentSandboxConfig) (*Manager, *int32, *int32) { + t.Helper() + cfg := map[string]config.AgentConfig{"deps": {Backend: "claude", Sandbox: override}} + m := NewManager(cfg, quietTestLogger(), ProjectContext{Org: "torch-spyre", Repos: []string{"spyre-inference"}}) + if global.WorkspaceDir == "" { + global.WorkspaceDir = t.TempDir() + } + m.SetSandboxConfig(global) + var podmanRuns, substrateRuns int32 + m.SetSandboxLauncher(countingLauncher{runs: &podmanRuns}) + m.setSandboxSubstrateLauncherFactoryForTest(func(sb config.SandboxSubstrateConfig) sandbox.Launcher { + if sb.Atespace != "hive-test" { + t.Errorf("factory got atespace=%q", sb.Atespace) + } + return countingLauncher{runs: &substrateRuns} + }) + m.setSandboxRunnerForTest(&sandboxFakeRunner{}) + if err := m.Start(context.Background(), "deps"); err != nil { + t.Fatalf("Start: %v", err) + } + return m, &podmanRuns, &substrateRuns +} + +func TestSubstrateRuntimeKickUsesTheSubstrateLauncher(t *testing.T) { + yes := true + m, podman, sbRuns := substrateRuntimeManager(t, + &config.AgentSandboxOverride{Enabled: &yes, Runtime: config.SandboxRuntimeSubstrate}, + config.AgentSandboxConfig{Enabled: true, Image: "dev-image", + Substrate: &config.SandboxSubstrateConfig{Atespace: "hive-test", Image: substrateTestImage}}) + if err := m.SendKick("deps", "bump the pinned dependency"); err != nil { + t.Fatalf("SendKick: %v", err) + } + waitIdle(t, m, "deps") + if got := atomic.LoadInt32(sbRuns); got != 1 { + t.Errorf("substrate launcher runs = %d, want 1", got) + } + if got := atomic.LoadInt32(podman); got != 0 { + t.Errorf("podman launcher ran %d time(s) for a substrate-runtime agent", got) + } +} + +func TestSubstrateRuntimeRefusesKickWithTagOnlyImage(t *testing.T) { + yes := true + m, _, sbRuns := substrateRuntimeManager(t, + &config.AgentSandboxOverride{Enabled: &yes, Runtime: config.SandboxRuntimeSubstrate}, + config.AgentSandboxConfig{Enabled: true, Image: "dev-image", + Substrate: &config.SandboxSubstrateConfig{Atespace: "hive-test", Image: "ghcr.io/x/y:latest"}}) + err := m.SendKick("deps", "fix") + if err == nil || !strings.Contains(err.Error(), "digest") { + t.Fatalf("SendKick error = %v, want a refusal naming the digest rule", err) + } + if got := atomic.LoadInt32(sbRuns); got != 0 { + t.Errorf("launcher ran %d time(s) despite the refusal", got) + } + if st := m.AllStatuses()["deps"].State; st == StateRunning { + t.Errorf("agent marked running after a refused kick") + } +} + +func TestSubstrateRuntimeRefusesCline(t *testing.T) { + yes := true + cfg := map[string]config.AgentConfig{"deps": {Backend: "cline", Sandbox: &config.AgentSandboxOverride{Enabled: &yes, Runtime: config.SandboxRuntimeSubstrate}}} + m := NewManager(cfg, quietTestLogger(), ProjectContext{Org: "torch-spyre", Repos: []string{"spyre-inference"}}) + m.SetSandboxConfig(config.AgentSandboxConfig{Enabled: true, Image: "dev-image", WorkspaceDir: t.TempDir(), + Substrate: &config.SandboxSubstrateConfig{Image: substrateTestImage}}) + m.setSandboxRunnerForTest(&sandboxFakeRunner{}) + if err := m.Start(context.Background(), "deps"); err != nil { + t.Fatalf("Start: %v", err) + } + err := m.SendKick("deps", "fix") + if err == nil || !strings.Contains(err.Error(), "Cline") { + t.Fatalf("SendKick error = %v, want the Cline refusal", err) + } +} + +func TestSubstrateOptionsFromConfig(t *testing.T) { + t.Setenv("SUBSTRATE_TEST_TOKEN", "sekrit") + opts := SubstrateOptionsFromConfig(config.SandboxSubstrateConfig{ + Endpoint: "api.ate-system.svc:443", + Atespace: "hive", + PolicyProfile: "strict", + TimeoutS: 90, + TokenEnv: "SUBSTRATE_TEST_TOKEN", + Image: substrateTestImage, + BridgeCommand: []string{"/bin/bridge"}, + AllowHosts: []string{"litellm.example.com"}, + WorkerSelector: map[string]string{"class": "gvisor"}, + SnapshotOnIdle: true, + }) + if opts.Token != "sekrit" || opts.Atespace != "hive" || opts.PolicyProfile != "strict" || opts.Timeout != 90*time.Second || !opts.SnapshotOnIdle { + t.Errorf("opts = %+v", opts) + } + if len(opts.Command) != 1 || opts.Command[0] != "/bin/bridge" || opts.WorkerSelector["class"] != "gvisor" { + t.Errorf("command/selector = %+v", opts) + } + if err := opts.Validate(); err != nil { + t.Errorf("Validate: %v", err) + } +} + +func TestSubstrateLauncherDefaultsToControlAPI(t *testing.T) { + m := NewManager(map[string]config.AgentConfig{}, quietTestLogger(), ProjectContext{}) + l := m.substrateLauncherLocked(config.SandboxSubstrateConfig{Atespace: "hive"}) + if _, ok := l.(*sandbox.SubstrateLauncher); !ok { + t.Fatalf("launcher = %T, want *sandbox.SubstrateLauncher", l) + } +} + +func TestSubstratePreflightWarningsDedupesControlPlanes(t *testing.T) { + // Two agents on the same control plane probe once. A CA file that does + // not exist fails the client constructor before any network I/O, which + // is reported as the control plane being unreachable. + blocks := []config.SandboxSubstrateConfig{ + {Endpoint: "127.0.0.1:1", Atespace: "dedupe-" + t.Name(), CAFile: "/nonexistent/ca.crt"}, + {Endpoint: "127.0.0.1:1", Atespace: "dedupe-" + t.Name(), CAFile: "/nonexistent/ca.crt"}, + } + got := substratePreflightWarnings(blocks) + if len(got) != 1 || !strings.Contains(got[0], "unreachable") { + t.Fatalf("warnings = %q, want exactly one unreachable warning for the shared control plane", got) + } +} diff --git a/src/crux/conditions.go b/src/crux/conditions.go index 3ee1942289..0175629224 100644 --- a/src/crux/conditions.go +++ b/src/crux/conditions.go @@ -37,9 +37,16 @@ const ( ClassDataDiskPressure = "data_disk_pressure" ClassGitHubAppTokenMintFailed = "github_app_token_mint_failed" ClassGitHubAppNotInstalled = "github_app_not_installed" - ClassAgentStall = "agent_stall" - ClassAgentCrashLoop = "agent_crash_loop" - ClassCIPollNudge = "ci_poll_nudge" + // ClassForgeUnreachable: a per-agent token mint failed at the transport + // layer (dial timeout, connection refused, context deadline) — the GitHub + // host cannot be reached from the pod. The App key and installation are + // not implicated, so this is kept apart from + // ClassGitHubAppTokenMintFailed, which would otherwise send the operator + // off to re-check App permissions during a network outage. + ClassForgeUnreachable = "forge_unreachable" + ClassAgentStall = "agent_stall" + ClassAgentCrashLoop = "agent_crash_loop" + ClassCIPollNudge = "ci_poll_nudge" // ClassRuntimeUnavailable is raised by the HOST, not the runtime: a // sidecar host whose runtime API stopped answering (hivecommons/hive#14138 // stage 4). The runtime cannot report its own absence. diff --git a/src/crux/config/agent_config.go b/src/crux/config/agent_config.go index 2033319af8..4047127a6f 100644 --- a/src/crux/config/agent_config.go +++ b/src/crux/config/agent_config.go @@ -435,6 +435,8 @@ type AgentSandboxConfig struct { Job *SandboxJobConfig `yaml:"job,omitempty" json:"job,omitempty"` // OpenShell holds the settings for sandbox.runtime: openshell (ADR-0026). OpenShell *SandboxOpenShellConfig `yaml:"openshell,omitempty" json:"openshell,omitempty"` + // Substrate holds the settings for sandbox.runtime: substrate (ADR-0026 phase 2). + Substrate *SandboxSubstrateConfig `yaml:"substrate,omitempty" json:"substrate,omitempty"` // Publish gates what a sandboxed run may publish; see SandboxPublishConfig. Publish *SandboxPublishConfig `yaml:"publish,omitempty" json:"publish,omitempty"` } @@ -476,6 +478,9 @@ type AgentSandboxOverride struct { // OpenShell, when set, overrides the global openshell block field by // field for this agent; see AgentConfig.SandboxOpenShell. OpenShell *SandboxOpenShellConfig `yaml:"openshell,omitempty" json:"openshell,omitempty"` + // Substrate, when set, overrides the global substrate block field by + // field for this agent; see AgentConfig.SandboxSubstrate. + Substrate *SandboxSubstrateConfig `yaml:"substrate,omitempty" json:"substrate,omitempty"` // Publish, when set, replaces the global publish preflight for this // agent; an empty verify list here turns the preflight off for it. Publish *SandboxPublishConfig `yaml:"publish,omitempty" json:"publish,omitempty"` diff --git a/src/crux/config/agent_config_substrate.go b/src/crux/config/agent_config_substrate.go new file mode 100644 index 0000000000..1a847d7d8e --- /dev/null +++ b/src/crux/config/agent_config_substrate.go @@ -0,0 +1,183 @@ +package config + +import ( + "strings" + "sync" +) + +var ( + substratePreflightMu sync.RWMutex + substratePreflight func([]SandboxSubstrateConfig) []string +) + +// SetSubstratePreflight registers the cluster-prerequisite check +// AgentSandboxGateWarnings runs for substrate-runtime agents. This package +// (and pkg/config above it) cannot import the Substrate client or +// crux/sandbox, so the code that owns the Control API client plugs itself in +// here. Passing nil unregisters it. +func SetSubstratePreflight(f func([]SandboxSubstrateConfig) []string) { + substratePreflightMu.Lock() + defer substratePreflightMu.Unlock() + substratePreflight = f +} + +// SubstratePreflightWarnings runs the registered preflight against the given +// substrate blocks; with none registered it reports nothing. +func SubstratePreflightWarnings(cfgs []SandboxSubstrateConfig) []string { + substratePreflightMu.RLock() + f := substratePreflight + substratePreflightMu.RUnlock() + if f == nil || len(cfgs) == 0 { + return nil + } + return f(cfgs) +} + +// SandboxSubstrateConfig is everything the Agent Substrate runtime needs +// beyond the agent's kick (ADR-0026 phase 2). Hive talks to Substrate's +// Control API (ate-api-server) over gRPC; Substrate owns the gVisor/microVM +// sandbox, the snapshot store and the atenet-egress gateway, so most of what +// the Job runtime spells out as a pod template is an ActorTemplate hive +// writes per kick from these settings. +// +// The workspace is NOT shared the way the Job and OpenShell runtimes share +// it: Substrate cannot mount an existing PersistentVolumeClaim, only a +// per-actor durableDir or a CSI volume it provisions itself. Hive therefore +// copies the per-kick workspace into the actor and back out through the +// bridge the sandbox image runs (crux substrate-bridge), reached over the +// atenet-router mTLS CONNECT tunnel. The local workspace stays the source of +// truth after the kick, so the push broker and PR flow are unchanged. +// +// AllowHosts is the egress allowlist handed to Substrate as the actor's +// EgressPolicy: the hosts hive's own MITM proxy (ADR-0002) would let a Podman +// sandbox reach — github.com, api.github.com — plus whatever the operator +// adds. Nothing else is reachable from the actor; Substrate default-denies. +// Credentials follow the Job rule: none of hive's enter it. Substrate's own +// egress credential injection is complementary and not used by hive. +type SandboxSubstrateConfig struct { + // Endpoint is the Control API address (host:port, or an http(s):// URL + // whose scheme picks TLS). Default api.ate-system.svc:443. + Endpoint string `yaml:"endpoint,omitempty" json:"endpoint,omitempty"` + // Atespace is the Substrate atespace (its tenant namespace, not a + // Kubernetes namespace) actors are created in. Default "hive". + Atespace string `yaml:"atespace,omitempty" json:"atespace,omitempty"` + // TokenEnv names the environment variable holding a Control API bearer + // token. Empty reads TokenFile instead. + TokenEnv string `yaml:"token_env,omitempty" json:"token_env,omitempty"` + // TokenFile is the projected ServiceAccount token (audience + // api.ate-system.svc) hive authenticates with. Default + // /var/run/secrets/ate/token. + TokenFile string `yaml:"token_file,omitempty" json:"token_file,omitempty"` + // CAFile is the PEM bundle the Control API serving certificate chains + // to (the servicedns ClusterTrustBundle projection). Empty uses + // /var/run/secrets/ate/ca.crt when present, else the system roots. + CAFile string `yaml:"ca_file,omitempty" json:"ca_file,omitempty"` + // InsecureTLS skips Control API and router certificate verification. + // Dev only. + InsecureTLS bool `yaml:"insecure_tls,omitempty" json:"insecure_tls,omitempty"` + // TimeoutS bounds one actor from create to delete. Non-positive uses + // the sandbox timeout (agent_sandbox.timeout_s) hive already applies to + // the kick. + TimeoutS int `yaml:"timeout_s,omitempty" json:"timeout_s,omitempty"` + // RouterURL is the atenet-router CONNECT listener hive reaches the + // actor's bridge through. Default https://atenet-router.ate-system.svc:8444. + RouterURL string `yaml:"router_url,omitempty" json:"router_url,omitempty"` + // RouterClientBundle and RouterCAFile are the podCertificate bundle the + // router authenticates hive with and the CA its serving certificate + // chains to. Defaults /run/router-client/credential-bundle.pem and + // /run/router-client/router-ca.pem. + RouterClientBundle string `yaml:"router_client_bundle,omitempty" json:"router_client_bundle,omitempty"` + RouterCAFile string `yaml:"router_ca_file,omitempty" json:"router_ca_file,omitempty"` + // BridgePort is the port the in-actor bridge listens on. Default 8477. + BridgePort int `yaml:"bridge_port,omitempty" json:"bridge_port,omitempty"` + // Image is the digest-pinned sandbox image (…@sha256:…) actors run. + // Substrate refuses tag-only references, so this overrides the agent + // sandbox image, which is usually a tag. + Image string `yaml:"image,omitempty" json:"image,omitempty"` + // BridgeCommand is the actor entrypoint that serves the bridge. Empty + // uses the crux binary's substrate-bridge subcommand. + BridgeCommand []string `yaml:"bridge_command,omitempty" json:"bridge_command,omitempty"` + // CPUMillicores and MemoryBytes are the ActorTemplate resource limits; + // Substrate requires both. Defaults 2000 and 4 GiB. + CPUMillicores int64 `yaml:"cpu_millicores,omitempty" json:"cpu_millicores,omitempty"` + MemoryBytes int64 `yaml:"memory_bytes,omitempty" json:"memory_bytes,omitempty"` + // WorkerSelector pins actors to WorkerPools carrying these labels; the + // pool decides the sandbox class (gvisor or microvm). Empty means any + // pool. + WorkerSelector map[string]string `yaml:"worker_selector,omitempty" json:"worker_selector,omitempty"` + // SnapshotOnIdle suspends the actor (asynchronous snapshot) after the + // kick instead of deleting it, so Substrate keeps a snapshot. The + // snapshot is retained until the agent's next idle snapshot replaces it + // (at most one per agent per hive process); it is not yet resumed on the + // next kick, and actors left from before a hive restart are not + // reclaimed. Default false: a kick is one-shot and snapshots cost + // object storage. + SnapshotOnIdle bool `yaml:"snapshot_on_idle,omitempty" json:"snapshot_on_idle,omitempty"` + // AllowHosts are additional egress hosts (host or host:port, leading + // *. wildcard allowed) the actor may reach, on top of the GitHub hosts + // hive always allows. + AllowHosts []string `yaml:"allow_hosts,omitempty" json:"allow_hosts,omitempty"` + // PolicyProfile names an operator-side profile; hive records it on the + // template so pools and in-actor tooling can act on it. + PolicyProfile string `yaml:"policy_profile,omitempty" json:"policy_profile,omitempty"` +} + +// SandboxSubstrate returns the effective Substrate runtime settings. The +// endpoint, atespace, credentials, TLS, router and bridge fields are cluster +// facts, so a per-agent block falls back to the global one for each it +// leaves empty; the image, resources, pool selector, snapshot choice and +// extra egress hosts are per-agent choices and replace the global ones +// wholesale when the per-agent block sets them. +func (a *AgentConfig) SandboxSubstrate(global AgentSandboxConfig) SandboxSubstrateConfig { + var g SandboxSubstrateConfig + if global.Substrate != nil { + g = *global.Substrate + } + if a == nil || a.Sandbox == nil || a.Sandbox.Substrate == nil { + return g + } + out := *a.Sandbox.Substrate + fallback := func(dst *string, src string) { + if strings.TrimSpace(*dst) == "" { + *dst = src + } + } + fallback(&out.Endpoint, g.Endpoint) + fallback(&out.Atespace, g.Atespace) + fallback(&out.TokenEnv, g.TokenEnv) + fallback(&out.TokenFile, g.TokenFile) + fallback(&out.CAFile, g.CAFile) + fallback(&out.RouterURL, g.RouterURL) + fallback(&out.RouterClientBundle, g.RouterClientBundle) + fallback(&out.RouterCAFile, g.RouterCAFile) + fallback(&out.Image, g.Image) + fallback(&out.PolicyProfile, g.PolicyProfile) + if out.TimeoutS == 0 { + out.TimeoutS = g.TimeoutS + } + if out.BridgePort == 0 { + out.BridgePort = g.BridgePort + } + if out.CPUMillicores == 0 { + out.CPUMillicores = g.CPUMillicores + } + if out.MemoryBytes == 0 { + out.MemoryBytes = g.MemoryBytes + } + if !out.InsecureTLS { + out.InsecureTLS = g.InsecureTLS + } + if !out.SnapshotOnIdle { + out.SnapshotOnIdle = g.SnapshotOnIdle + } + if len(out.BridgeCommand) == 0 { + out.BridgeCommand = g.BridgeCommand + } + if len(out.AllowHosts) == 0 { + out.AllowHosts = g.AllowHosts + } + if len(out.WorkerSelector) == 0 { + out.WorkerSelector = g.WorkerSelector + } + return out +} diff --git a/src/crux/config/agent_config_substrate_test.go b/src/crux/config/agent_config_substrate_test.go new file mode 100644 index 0000000000..5b55bc2998 --- /dev/null +++ b/src/crux/config/agent_config_substrate_test.go @@ -0,0 +1,121 @@ +package config + +import ( + "reflect" + "testing" +) + +func TestSandboxSubstrateFallsBackToGlobal(t *testing.T) { + var nilA *AgentConfig + if got := nilA.SandboxSubstrate(AgentSandboxConfig{}); !reflect.DeepEqual(got, SandboxSubstrateConfig{}) { + t.Fatalf("nil agent, nil global = %+v, want zero value", got) + } + + global := AgentSandboxConfig{Substrate: &SandboxSubstrateConfig{ + Endpoint: "api.ate-system.svc:443", Atespace: "hive", TokenEnv: "ATE_TOKEN", TokenFile: "/tok", + CAFile: "/ca", InsecureTLS: true, TimeoutS: 900, RouterURL: "https://router:8444", + RouterClientBundle: "/bundle", RouterCAFile: "/router-ca", BridgePort: 9000, + Image: "img@sha256:abc", BridgeCommand: []string{"/bridge"}, CPUMillicores: 1000, MemoryBytes: 1 << 30, + WorkerSelector: map[string]string{"class": "gvisor"}, SnapshotOnIdle: true, + AllowHosts: []string{"hub.example"}, PolicyProfile: "strict", + }} + if got := nilA.SandboxSubstrate(global); !reflect.DeepEqual(got, *global.Substrate) { + t.Fatalf("nil agent = %+v, want global block", got) + } + noBlock := &AgentConfig{Sandbox: &AgentSandboxOverride{Image: "a-img"}} + if got := noBlock.SandboxSubstrate(global); !reflect.DeepEqual(got, *global.Substrate) { + t.Fatalf("agent without substrate block = %+v, want global block", got) + } + + // A per-agent block that names nothing but whitespace inherits every + // field from the global one. + empty := &AgentConfig{Sandbox: &AgentSandboxOverride{Substrate: &SandboxSubstrateConfig{Endpoint: " "}}} + if got := empty.SandboxSubstrate(global); !reflect.DeepEqual(got, *global.Substrate) { + t.Fatalf("blank per-agent block = %+v, want global block", got) + } +} + +func TestSandboxSubstratePerAgentFieldsWin(t *testing.T) { + global := AgentSandboxConfig{Substrate: &SandboxSubstrateConfig{ + Endpoint: "api-global:443", Atespace: "global", TokenEnv: "GLOBAL_TOKEN", TokenFile: "/global-tok", + CAFile: "/global-ca", TimeoutS: 900, RouterURL: "https://router-global:8444", + RouterClientBundle: "/global-bundle", RouterCAFile: "/global-router-ca", BridgePort: 8477, + Image: "global@sha256:aaa", BridgeCommand: []string{"/global-bridge"}, CPUMillicores: 2000, MemoryBytes: 4 << 30, + WorkerSelector: map[string]string{"pool": "global"}, + AllowHosts: []string{"global.example", "hub.example"}, PolicyProfile: "global-policy", + }} + a := &AgentConfig{Sandbox: &AgentSandboxOverride{Substrate: &SandboxSubstrateConfig{ + Endpoint: "api-agent:443", TimeoutS: 60, InsecureTLS: true, BridgePort: 9001, + Image: "agent@sha256:bbb", BridgeCommand: []string{"/agent-bridge"}, CPUMillicores: 500, MemoryBytes: 1 << 30, + WorkerSelector: map[string]string{"gpu": "a100"}, SnapshotOnIdle: true, + AllowHosts: []string{"models.example"}, + }}} + got := a.SandboxSubstrate(global) + + if got.Endpoint != "api-agent:443" || got.TimeoutS != 60 || !got.InsecureTLS || got.BridgePort != 9001 || got.Image != "agent@sha256:bbb" { + t.Fatalf("per-agent fields not kept: %+v", got) + } + if got.Atespace != "global" || got.TokenEnv != "GLOBAL_TOKEN" || got.TokenFile != "/global-tok" || got.CAFile != "/global-ca" || + got.RouterURL != "https://router-global:8444" || got.RouterClientBundle != "/global-bundle" || got.RouterCAFile != "/global-router-ca" || got.PolicyProfile != "global-policy" { + t.Fatalf("empty per-agent cluster facts not inherited: %+v", got) + } + if got.CPUMillicores != 500 || got.MemoryBytes != 1<<30 || !got.SnapshotOnIdle { + t.Fatalf("per-agent resources/snapshot not kept: %+v", got) + } + if !reflect.DeepEqual(got.AllowHosts, []string{"models.example"}) { + t.Fatalf("allow_hosts = %v, want per-agent list only", got.AllowHosts) + } + if !reflect.DeepEqual(got.BridgeCommand, []string{"/agent-bridge"}) { + t.Fatalf("bridge_command = %v, want per-agent list only", got.BridgeCommand) + } + if !reflect.DeepEqual(got.WorkerSelector, map[string]string{"gpu": "a100"}) { + t.Fatalf("worker_selector = %v, want per-agent map only", got.WorkerSelector) + } + + // Booleans: a global true cannot be switched off per-agent. + global.Substrate.InsecureTLS = true + global.Substrate.SnapshotOnIdle = true + a.Sandbox.Substrate.InsecureTLS = false + a.Sandbox.Substrate.SnapshotOnIdle = false + if merged := a.SandboxSubstrate(global); !merged.InsecureTLS || !merged.SnapshotOnIdle { + t.Fatal("per-agent false inherits global true") + } + + // The merge must not write the inherited values back into its inputs. + if global.Substrate.AllowHosts[0] != "global.example" || a.Sandbox.Substrate.Atespace != "" { + t.Fatal("merge mutated its inputs") + } +} + +func TestSubstratePreflightRegistry(t *testing.T) { + t.Cleanup(func() { SetSubstratePreflight(nil) }) + cfgs := []SandboxSubstrateConfig{{Endpoint: "api:443"}} + + SetSubstratePreflight(nil) + if got := SubstratePreflightWarnings(cfgs); got != nil { + t.Fatalf("unregistered preflight = %v, want nil", got) + } + + var seen []SandboxSubstrateConfig + SetSubstratePreflight(func(c []SandboxSubstrateConfig) []string { + seen = c + return []string{"control api unreachable"} + }) + if got := SubstratePreflightWarnings(nil); got != nil || seen != nil { + t.Fatalf("empty cfgs = %v (called=%v), want nil and no call", got, seen != nil) + } + if got := SubstratePreflightWarnings(cfgs); !reflect.DeepEqual(got, []string{"control api unreachable"}) || !reflect.DeepEqual(seen, cfgs) { + t.Fatalf("registered preflight = %v (seen %v), want its warnings for the given cfgs", got, seen) + } + + SetSubstratePreflight(nil) + if got := SubstratePreflightWarnings(cfgs); got != nil { + t.Fatalf("after unregister = %v, want nil", got) + } +} + +func TestValidSandboxRuntimesIncludesSubstrate(t *testing.T) { + if !ValidSandboxRuntimes[SandboxRuntimeSubstrate] { + t.Fatal("substrate is not a valid sandbox runtime") + } +} diff --git a/src/crux/config/sandbox_runtime.go b/src/crux/config/sandbox_runtime.go index 994c45d788..9fa0e5a342 100644 --- a/src/crux/config/sandbox_runtime.go +++ b/src/crux/config/sandbox_runtime.go @@ -7,10 +7,11 @@ const ( SandboxRuntimePodman = "podman" SandboxRuntimeJob = "job" SandboxRuntimeOpenShell = "openshell" + SandboxRuntimeSubstrate = "substrate" ) // ValidSandboxRuntimes is the closed set a config may name. -var ValidSandboxRuntimes = map[string]bool{SandboxRuntimePodman: true, SandboxRuntimeJob: true, SandboxRuntimeOpenShell: true} +var ValidSandboxRuntimes = map[string]bool{SandboxRuntimePodman: true, SandboxRuntimeJob: true, SandboxRuntimeOpenShell: true, SandboxRuntimeSubstrate: true} // SandboxJobConfig is everything the Job runtime needs beyond the image // (#6311). The pod-template fields pass through to Kubernetes verbatim. diff --git a/src/crux/effects/effects.go b/src/crux/effects/effects.go index 3c8c4ffb72..df06507e5b 100644 --- a/src/crux/effects/effects.go +++ b/src/crux/effects/effects.go @@ -43,6 +43,12 @@ var ErrNotApplied = errors.New("external effect definitely not applied") // must be reconciled (see Reconcile) before any retry. var ErrNeedsReconciliation = errors.New("external effect needs reconciliation before retry") +// ErrUnresolvedExpired accompanies ErrNeedsReconciliation when the boundary +// refused this attempt but, finding the operation unresolved past its TTL, +// expired it to not-applied: the effect did not run, and the caller's next +// attempt (after its own duplicate check) may. +var ErrUnresolvedExpired = errors.New("unresolved external effect expired past TTL; retry authorized") + // NotApplied wraps err as a definite refusal (ErrNotApplied) while keeping // err's own message and chain, so callers still see the original error. func NotApplied(err error) error { @@ -146,6 +152,7 @@ func ExecuteReconciling(ctx context.Context, boundary Boundary, claim Claim, obs if expErr != nil { return Result{}, fmt.Errorf("%w (reconciliation lookup failed: %v; expiry failed: %v)", journalErr, lookupErr, expErr) } + expired = expired || errors.Is(journalErr, ErrUnresolvedExpired) if !expired { return Result{}, fmt.Errorf("%w (reconciliation lookup failed: %v)", journalErr, lookupErr) } diff --git a/src/crux/effects/execute_reconciling_test.go b/src/crux/effects/execute_reconciling_test.go index 24d473bf07..558ca6b3d4 100644 --- a/src/crux/effects/execute_reconciling_test.go +++ b/src/crux/effects/execute_reconciling_test.go @@ -3,6 +3,7 @@ package effects import ( "context" "errors" + "fmt" "strings" "testing" "time" @@ -192,3 +193,31 @@ func TestLoggingBoundaryLookupAppliedForwards(t *testing.T) { t.Fatal("LoggingBoundary over a non-Lookup must not report applied") } } + +// A boundary that already expired the operation while refusing (the refusal +// carries ErrUnresolvedExpired) authorizes the retry even though its own +// ExpireUnresolved now finds nothing unresolved to expire. +func TestExecuteReconcilingBoundaryExpiredRefusalRetries(t *testing.T) { + b := &expiredOnRefusalBoundary{} + ran := 0 + res, err := ExecuteReconciling(context.Background(), b, reconcilingClaim, + func(context.Context) (ExternalState, error) { return ExternalState{}, errors.New("api down") }, + time.Minute, okEffect(&ran)) + if err != nil || ran != 1 || res.Provenance != "ran" { + t.Fatalf("got (%+v, %v, ran=%d), want the effect to run once", res, err, ran) + } +} + +type expiredOnRefusalBoundary struct{ refused bool } + +func (b *expiredOnRefusalBoundary) Execute(ctx context.Context, _ Claim, effect func(context.Context) (Result, error)) (Result, error) { + if !b.refused { + b.refused = true + return Result{}, fmt.Errorf("%w: %w", ErrNeedsReconciliation, ErrUnresolvedExpired) + } + return effect(ctx) +} + +func (b *expiredOnRefusalBoundary) ExpireUnresolved(context.Context, Claim, time.Duration) (bool, error) { + return false, nil +} diff --git a/src/crux/egressmark/egressmark.go b/src/crux/egressmark/egressmark.go new file mode 100644 index 0000000000..0d70e8fb1d --- /dev/null +++ b/src/crux/egressmark/egressmark.go @@ -0,0 +1,82 @@ +// Package egressmark stamps the hive's OWN outbound sockets with the fwmark +// (SO_MARK) that the entrypoint's forced-egress iptables rule exempts +// (`-m mark --mark -j RETURN`), so control-plane traffic the hive +// itself originates is never redirected into its own MITM proxy. +// +// It is shared by the proxy's upstream dials and by every in-cluster +// Kubernetes API client (self-upgrade, startup-probe backfill, node metrics, +// ingress probes, kubejob). The entrypoint also exempts the hive's uid with +// `-m owner`, but that match needs the xt_owner kernel module, which IBM Cloud +// IKS/ROKS and other OVN nodes do not load: there the mark is the ONLY +// exemption, and an unmarked dial to kubernetes.default.svc:443 lands in the +// proxy, whose private-range forward guard closes it ("EOF" / "unexpected eof +// while reading"), failing every spoke self-upgrade on that cluster. +package egressmark + +import ( + "log/slog" + "net" + "os" + "strconv" + "strings" + "sync" + "syscall" + "time" +) + +const ( + // DefaultMark is the fwmark the entrypoint exempts when + // HIVE_PROXY_EGRESS_MARK is unset. entrypoint.sh is the single source of + // truth and exports the same value into the hive process. + DefaultMark = 0x1112 + + // Env is the variable entrypoint.sh exports so the Go side and the + // iptables rule stay in lockstep. + Env = "HIVE_PROXY_EGRESS_MARK" +) + +// Mark returns the fwmark to stamp on the hive's own dials, read from +// HIVE_PROXY_EGRESS_MARK (hex with 0x prefix or decimal). It falls back to +// DefaultMark when unset or unparseable: that default matches the entrypoint +// default, so accidental drift stays on the agreed value rather than silently +// disabling the exemption. +func Mark() int { + raw := strings.TrimSpace(os.Getenv(Env)) + if raw == "" { + return DefaultMark + } + if v, err := strconv.ParseInt(raw, 0, 64); err == nil && v > 0 { + return int(v) + } + return DefaultMark +} + +// Dialer returns a *net.Dialer whose Control hook stamps Mark() on the socket +// before connect. The optional timeout is applied when non-zero. +// +// The mark is BEST-EFFORT: setsockopt(SO_MARK) needs CAP_NET_ADMIN in the +// effective set, and a hive running without it must still dial (on OKE the +// owner-uid exemption covers it). A mark failure is therefore logged once and +// the dial proceeds unmarked instead of aborting. +func Dialer(timeout time.Duration) *net.Dialer { + mark := Mark() + return &net.Dialer{ + Timeout: timeout, + Control: func(_, _ string, c syscall.RawConn) error { + return c.Control(func(fd uintptr) { + if err := setSockMark(fd, mark); err != nil { + warnOnce(mark, err) + } + }) + }, + } +} + +var warnSync sync.Once + +func warnOnce(mark int, err error) { + warnSync.Do(func() { + slog.Warn("egress SO_MARK unavailable — dialing unmarked (owner-UID exemption still applies where xt_owner is loaded); expected when the hive runs without CAP_NET_ADMIN in its effective set", + "mark", mark, "error", err) + }) +} diff --git a/src/crux/egressmark/egressmark_test.go b/src/crux/egressmark/egressmark_test.go new file mode 100644 index 0000000000..9c3ed7040d --- /dev/null +++ b/src/crux/egressmark/egressmark_test.go @@ -0,0 +1,84 @@ +package egressmark + +import ( + "errors" + "net" + "syscall" + "testing" + "time" +) + +// TestMarkDefaultAndEnvOverride verifies the mark resolves to the entrypoint +// default when unset, honours hex and decimal overrides, and fails safe back +// to the default on garbage or zero rather than disabling the exemption. +func TestMarkDefaultAndEnvOverride(t *testing.T) { + cases := []struct { + name string + env string + want int + }{ + {"unset", "", DefaultMark}, + {"hex", "0x2223", 0x2223}, + {"decimal", "8739", 8739}, + {"garbage falls back", "not-a-number", DefaultMark}, + {"zero falls back", "0", DefaultMark}, + {"negative falls back", "-5", DefaultMark}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Setenv(Env, tc.env) + if got := Mark(); got != tc.want { + t.Fatalf("Mark() with %s=%q = %#x, want %#x", Env, tc.env, got, tc.want) + } + }) + } +} + +// TestDialerControlDoesNotBreakDial dials a loopback listener through the +// marked dialer. The Control hook must run and attempt SO_MARK without failing +// the dial: the hive runs without CAP_NET_ADMIN in its effective set in many +// deployments, so setsockopt(SO_MARK) returns EPERM there and a propagated +// error would break every self-upgrade / startup-probe / kubejob API call. +func TestDialerControlDoesNotBreakDial(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + go func() { + c, err := ln.Accept() + if err != nil { + return + } + c.Close() + }() + + d := Dialer(2 * time.Second) + if d.Timeout != 2*time.Second { + t.Fatalf("Dialer timeout = %v, want 2s", d.Timeout) + } + conn, err := d.Dial("tcp", ln.Addr().String()) + if err != nil { + if errors.Is(err, syscall.EPERM) { + t.Fatalf("Dialer aborted the dial on SO_MARK EPERM — mark must be best-effort: %v", err) + } + t.Fatalf("Dialer Dial failed (Control hook must not break the dial): %v", err) + } + conn.Close() +} + +// TestDialerZeroTimeoutLeavesDefault confirms a zero timeout is passed through +// unchanged so callers that rely on the http.Transport's own deadlines are not +// given a surprise dial timeout. +func TestDialerZeroTimeoutLeavesDefault(t *testing.T) { + if d := Dialer(0); d.Timeout != 0 || d.Control == nil { + t.Fatalf("Dialer(0) = timeout %v, control nil=%v; want 0 and a Control hook", d.Timeout, d.Control == nil) + } +} + +// TestWarnOnceIsIdempotent exercises the once-only warning path directly so a +// mark failure on every dial does not spam the log. +func TestWarnOnceIsIdempotent(t *testing.T) { + warnOnce(DefaultMark, errors.New("first")) + warnOnce(DefaultMark, errors.New("second")) +} diff --git a/src/crux/egressmark/somark_linux.go b/src/crux/egressmark/somark_linux.go new file mode 100644 index 0000000000..0e7ca6e503 --- /dev/null +++ b/src/crux/egressmark/somark_linux.go @@ -0,0 +1,11 @@ +//go:build linux + +package egressmark + +import "syscall" + +// setSockMark sets SO_MARK on the raw socket so the kernel stamps every packet +// it emits with the fwmark the entrypoint's nat/OUTPUT RETURN rule matches. +func setSockMark(fd uintptr, mark int) error { + return syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_MARK, mark) +} diff --git a/src/crux/egressmark/somark_other.go b/src/crux/egressmark/somark_other.go new file mode 100644 index 0000000000..9cb20460a3 --- /dev/null +++ b/src/crux/egressmark/somark_other.go @@ -0,0 +1,9 @@ +//go:build !linux + +package egressmark + +// setSockMark is a no-op off Linux: SO_MARK is Linux-only, but the package must +// still compile on the CI matrix and developer machines. +func setSockMark(fd uintptr, mark int) error { + return nil +} diff --git a/src/crux/go.mod b/src/crux/go.mod index ec82f74fff..f9e6c9f12a 100644 --- a/src/crux/go.mod +++ b/src/crux/go.mod @@ -16,6 +16,8 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.47.0 go.opentelemetry.io/otel/trace v1.47.0 golang.org/x/sys v0.49.0 + google.golang.org/grpc v1.85.0 + google.golang.org/protobuf v1.36.12 gopkg.in/yaml.v3 v3.0.1 ) @@ -50,6 +52,4 @@ require ( golang.org/x/time v0.16.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260928230214-8a89bd6388cc // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260928230214-8a89bd6388cc // indirect - google.golang.org/grpc v1.85.0 // indirect - google.golang.org/protobuf v1.36.12 // indirect ) diff --git a/src/crux/kubejob/kubejob.go b/src/crux/kubejob/kubejob.go index 7744e75582..5ccc38c61a 100644 --- a/src/crux/kubejob/kubejob.go +++ b/src/crux/kubejob/kubejob.go @@ -41,6 +41,7 @@ import ( "strings" "time" + "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/crux/sandbox" ) @@ -782,6 +783,6 @@ func (l *Launcher) client() (*http.Client, error) { if !pool.AppendCertsFromPEM(pem) { return nil, errors.New("kubejob: in-cluster CA bundle contained no certificates") } - l.HTTPClient = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}} + l.HTTPClient = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}, DialContext: egressmark.Dialer(0).DialContext}} return l.HTTPClient, nil } diff --git a/src/crux/proxy/github_proxy.go b/src/crux/proxy/github_proxy.go index 35a1ad0844..c47abfedf8 100644 --- a/src/crux/proxy/github_proxy.go +++ b/src/crux/proxy/github_proxy.go @@ -11,15 +11,14 @@ import ( "net" "net/http" "os" - "strconv" "strings" "sync" - "syscall" "time" "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/agent" "github.com/hivecommons/hive/crux/config" + "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/crux/inferencehealth" "github.com/hivecommons/hive/crux/upstreamproxy" ) @@ -38,94 +37,26 @@ const ( maxViolationLog = 1000 maxGitHubWriteBodyScan = 2 << 20 - // defaultProxyEgressMark is the fwmark (SO_MARK) the proxy stamps onto its - // OWN upstream :443 dials so the forced-egress iptables redirect can exempt - // them (via `-m mark --mark -j RETURN`) and avoid looping the proxy's - // traffic back into itself. - // - // This replaces the `-m owner --uid-owner` self-exemption, which needs the - // xt_owner kernel module — absent on OpenShift/OVN nodes. A packet mark works - // on both OKE and OpenShift. The value is arbitrary but MUST match the mark - // the entrypoint exempts; entrypoint.sh is the single source of truth and - // exports HIVE_PROXY_EGRESS_MARK (defaulting to this same value) into the - // proxy process, which reads it via proxyEgressMark(). - defaultProxyEgressMark = 0x1112 - - // proxyEgressMarkEnv is the env var the entrypoint exports to keep the proxy - // and the iptables rule in sync from one source of truth. - proxyEgressMarkEnv = "HIVE_PROXY_EGRESS_MARK" + // defaultProxyEgressMark and proxyEgressMarkEnv mirror crux/egressmark, + // the shared home of the SO_MARK self-exemption (also used by the hive's + // in-cluster Kubernetes API clients). entrypoint.sh remains the single + // source of truth for the value. + defaultProxyEgressMark = egressmark.DefaultMark + proxyEgressMarkEnv = egressmark.Env ) -// proxyEgressMark returns the fwmark to stamp on the proxy's upstream dials, -// read from HIVE_PROXY_EGRESS_MARK (set by entrypoint.sh) so the Go proxy and -// the iptables exemption stay in lockstep. Accepts hex (0x1112) or decimal. -// Falls back to defaultProxyEgressMark when unset or unparseable — that default -// matches the entrypoint default, so an accidental drift fails safe by staying -// on the agreed value rather than silently disabling the exemption. +// proxyEgressMark returns the fwmark to stamp on the proxy's upstream dials; +// see egressmark.Mark. func proxyEgressMark() int { - raw := strings.TrimSpace(os.Getenv(proxyEgressMarkEnv)) - if raw == "" { - return defaultProxyEgressMark - } - // ParseInt with base 0 honours a 0x prefix (hex) and plain decimal. - if v, err := strconv.ParseInt(raw, 0, 64); err == nil && v > 0 { - return int(v) - } - return defaultProxyEgressMark + return egressmark.Mark() } -// markDialer returns a *net.Dialer whose Control hook stamps the proxy egress -// fwmark (SO_MARK) on the outbound socket before connect. On non-Linux builds -// setSockMark is a no-op, so the dialer is portable. The optional timeout is -// applied when non-zero. -// -// SO_MARK is a BEST-EFFORT egress hint, never a hard requirement: it exists so -// the forced-egress iptables redirect can exempt the proxy's own upstream dials -// via `-m mark`. But setsockopt(SO_MARK) needs CAP_NET_ADMIN in the process's -// EFFECTIVE set, and the hive Go process runs as a non-root user (entrypoint -// drops from root to `dev` via gosu, which strips all capabilities). So the mark -// call fails with EPERM ("operation not permitted") on exactly the deployments -// that run non-root — which is all of them. Returning that error from the -// Control hook ABORTS the dial, so every proxy upstream dial fails and agents -// can't reach GitHub at all. That is a regression: on OKE the `-m owner -// --uid-owner` exemption already covers the proxy's uid dials, so an unmarked -// dial succeeds; on OpenShift (no xt_owner) the mark is the only exemption, but -// there too, failing the dial helps nothing. So we log the mark failure ONCE and -// proceed unmarked rather than killing the connection. +// markDialer returns a *net.Dialer whose sockets carry the proxy egress fwmark +// so the forced-egress REDIRECT exempts them. The mark is best-effort: without +// CAP_NET_ADMIN the dial proceeds unmarked (owner-UID exemption on OKE); see +// egressmark.Dialer for the full rationale. func markDialer(timeout time.Duration) *net.Dialer { - mark := proxyEgressMark() - return &net.Dialer{ - Timeout: timeout, - Control: func(_, _ string, c syscall.RawConn) error { - // A failure to enter the raw-conn control context is a real dial - // problem (the fd is unusable) — propagate it. - if err := c.Control(func(fd uintptr) { - if markErr := setSockMark(fd, mark); markErr != nil { - warnSockMarkOnce(mark, markErr) - } - }); err != nil { - return err - } - // Deliberately do NOT propagate a setSockMark failure: the mark is a - // best-effort exemption hint, and the owner-UID iptables RETURN - // already exempts the proxy's own uid on OKE, so an unmarked dial - // still reaches GitHub. - return nil - }, - } -} - -// warnSockMarkOnce logs the first SO_MARK failure at WARN and every subsequent -// one at DEBUG, so a non-CAP_NET_ADMIN deployment (the common non-root case) -// does not flood the log with one line per dial while still surfacing the -// condition once. -var sockMarkWarnOnce sync.Once - -func warnSockMarkOnce(mark int, err error) { - sockMarkWarnOnce.Do(func() { - slog.Warn("proxy egress SO_MARK unavailable — dialing unmarked (owner-UID exemption still applies on OKE); this is expected when the hive runs non-root without CAP_NET_ADMIN in its effective set", - "mark", mark, "error", err) - }) + return egressmark.Dialer(timeout) } // CACertPath and caKeyPath are the PVC locations of the persisted MITM CA. diff --git a/src/crux/proxy/github_proxy_inference.go b/src/crux/proxy/github_proxy_inference.go index 1d22313c5d..59a2ef23ec 100644 --- a/src/crux/proxy/github_proxy_inference.go +++ b/src/crux/proxy/github_proxy_inference.go @@ -397,6 +397,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { if !ok { p.logger.Warn("inference request from unidentified caller refused", p.translatorRefusalAttrs(r)...) + setTranslatorOutcome(r.Context(), outcomeUnidentifiedCaller) http.Error(w, `{"type":"error","error":{"type":"authentication_error","message":"caller could not be identified"}}`, http.StatusUnauthorized) return } @@ -412,6 +413,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { route := p.inference.Get(agentName) if route == nil { p.logger.Warn("inference no route", "agent", agentName, "reason", p.inference.MissReason(agentName)) + setTranslatorOutcome(r.Context(), outcomeNoRoute) http.Error(w, `{"type":"error","error":{"type":"api_error","message":"no inference route for agent"}}`, http.StatusBadGateway) return } @@ -585,7 +587,7 @@ func (p *GitHubProxy) inferenceTranslatorHandler() http.Handler { _, _ = w.Write(translated) }) - return mux + return instrumentTranslator(mux) } // StartInferenceTranslator runs the Anthropic-compatible inference translator diff --git a/src/crux/proxy/metrics.go b/src/crux/proxy/metrics.go new file mode 100644 index 0000000000..bc7c4e0311 --- /dev/null +++ b/src/crux/proxy/metrics.go @@ -0,0 +1,164 @@ +package proxy + +import ( + "context" + "net/http" + "sync" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" +) + +// Inference translator instruments (hivecommons/crux#153). Like serve's they +// sit on the global providers, which are OTel's delegating no-ops until a host +// registers an SDK provider, so recording costs nothing by default. +// +// Attributes are bounded to the status class and a fixed outcome enum. Never an +// agent name, path, model name, token or body. +const ( + instrumentationName = "github.com/hivecommons/hive/crux/proxy" + + translatorDurationMetric = "crux.proxy.inference.duration" + translatorCountMetric = "crux.proxy.inference.requests" +) + +// Outcome of one translator request: a fixed enum, so the attribute stays bounded. +const ( + outcomeOK = "ok" + outcomeUnidentifiedCaller = "unidentified_caller" + outcomeNoRoute = "no_route" + outcomeUpstreamError = "upstream_error" +) + +var ( + attrStatusClass = attribute.Key("status_class") + attrOutcome = attribute.Key("outcome") +) + +type translatorMetrics struct { + duration metric.Float64Histogram + requests metric.Int64Counter +} + +func newTranslatorMetrics(meter metric.Meter) translatorMetrics { + // Instrument constructors only fail on an invalid name or unit; both are + // constants here, and the returned instrument is a usable no-op either way. + duration, _ := meter.Float64Histogram(translatorDurationMetric, + metric.WithUnit("s"), + metric.WithDescription("Time the inference translator took to answer a request.")) + requests, _ := meter.Int64Counter(translatorCountMetric, + metric.WithDescription("Inference translator requests, by status class and outcome.")) + return translatorMetrics{duration: duration, requests: requests} +} + +var globalTranslatorMetrics = sync.OnceValue(func() translatorMetrics { + return newTranslatorMetrics(otel.Meter(instrumentationName)) +}) + +func (m translatorMetrics) record(ctx context.Context, d time.Duration, attrs []attribute.KeyValue) { + opt := metric.WithAttributes(attrs...) + m.duration.Record(ctx, d.Seconds(), opt) + m.requests.Add(ctx, 1, opt) +} + +type outcomeKey struct{} + +// outcomeHolder lets the handler name an outcome the status alone cannot tell. +type outcomeHolder struct{ outcome string } + +func setTranslatorOutcome(ctx context.Context, outcome string) { + if h, ok := ctx.Value(outcomeKey{}).(*outcomeHolder); ok { + h.outcome = outcome + } +} + +type translatorStatusRecorder struct { + http.ResponseWriter + status int +} + +func (w *translatorStatusRecorder) WriteHeader(status int) { + if w.status == 0 { + w.status = status + } + w.ResponseWriter.WriteHeader(status) +} + +func (w *translatorStatusRecorder) Write(b []byte) (int, error) { + if w.status == 0 { + w.status = http.StatusOK + } + return w.ResponseWriter.Write(b) +} + +func (w *translatorStatusRecorder) Unwrap() http.ResponseWriter { return w.ResponseWriter } + +// translatorFlushRecorder offers Flush only when the wrapped writer does; the +// SSE path type-asserts http.Flusher. +type translatorFlushRecorder struct { + *translatorStatusRecorder + http.Flusher +} + +func translatorStatusClass(status int) string { + switch { + case status >= 500: + return "5xx" + case status >= 400: + return "4xx" + case status >= 300: + return "3xx" + case status >= 200: + return "2xx" + default: + return "1xx" + } +} + +// instrumentTranslator records one request's server span, duration and count +// around next. +func instrumentTranslator(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + start := time.Now() + holder := &outcomeHolder{} + ctx, span := otel.Tracer(instrumentationName).Start( + context.WithValue(r.Context(), outcomeKey{}, holder), + "inference_translator", trace.WithSpanKind(trace.SpanKindServer)) + rec := &translatorStatusRecorder{ResponseWriter: w} + var out http.ResponseWriter = rec + if f, ok := w.(http.Flusher); ok { + out = translatorFlushRecorder{rec, f} + } + returned := false + defer func() { + status := rec.status + if status == 0 { + status = http.StatusOK + } + if !returned { + status = http.StatusInternalServerError + span.SetStatus(codes.Error, "handler panic") + } + outcome := holder.outcome + if outcome == "" { + outcome = outcomeOK + if status >= 400 { + outcome = outcomeUpstreamError + } + } + attrs := []attribute.KeyValue{ + attrStatusClass.String(translatorStatusClass(status)), + attrOutcome.String(outcome), + } + span.SetAttributes(attrs...) + span.End() + globalTranslatorMetrics().record(ctx, time.Since(start), attrs) + }() + next.ServeHTTP(out, r.WithContext(ctx)) + returned = true + }) +} diff --git a/src/crux/proxy/metrics_test.go b/src/crux/proxy/metrics_test.go new file mode 100644 index 0000000000..0fcb9b8df4 --- /dev/null +++ b/src/crux/proxy/metrics_test.go @@ -0,0 +1,89 @@ +package proxy + +import ( + "context" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "testing" + + "github.com/hivecommons/hive/crux/agent" + + "go.opentelemetry.io/otel/attribute" + sdkmetric "go.opentelemetry.io/otel/sdk/metric" + "go.opentelemetry.io/otel/sdk/metric/metricdata" +) + +func TestTranslatorMetricsBoundedAttributes(t *testing.T) { + fakeProcNetTCP(t) + reader := sdkmetric.NewManualReader() + mp := sdkmetric.NewMeterProvider(sdkmetric.WithReader(reader)) + m := newTranslatorMetrics(mp.Meter(instrumentationName)) + prev := globalTranslatorMetrics + globalTranslatorMetrics = func() translatorMetrics { return m } + t.Cleanup(func() { + globalTranslatorMetrics = prev + _ = mp.Shutdown(context.Background()) + }) + + p := &GitHubProxy{logger: slog.New(slog.NewTextHandler(io.Discard, nil))} + p.uidMap = &agent.UIDMap{BaseUID: 2001, Agents: map[string]int{"quality": 2002}} + rec := httptest.NewRecorder() + p.inferenceTranslatorHandler().ServeHTTP(rec, translatorReq("127.0.0.1:43210", "quality")) + if rec.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401", rec.Code) + } + + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("Collect: %v", err) + } + var counts, durations int + for _, sm := range rm.ScopeMetrics { + for _, mt := range sm.Metrics { + got := map[string]string{} + collect := func(len int, kv func(string) (string, bool)) { + if len != 2 { + t.Errorf("%s: %d attributes, want exactly status_class and outcome", mt.Name, len) + } + for _, k := range []string{"status_class", "outcome", "agent", "path", "model"} { + if v, ok := kv(k); ok { + got[k] = v + } + } + } + switch data := mt.Data.(type) { + case metricdata.Sum[int64]: + counts++ + for _, dp := range data.DataPoints { + collect(dp.Attributes.Len(), func(k string) (string, bool) { + v, ok := dp.Attributes.Value(attrKey(k)) + return v.AsString(), ok + }) + } + case metricdata.Histogram[float64]: + durations++ + for _, dp := range data.DataPoints { + collect(dp.Attributes.Len(), func(k string) (string, bool) { + v, ok := dp.Attributes.Value(attrKey(k)) + return v.AsString(), ok + }) + } + } + if got["status_class"] != "4xx" || got["outcome"] != outcomeUnidentifiedCaller { + t.Errorf("%s attributes = %v, want 4xx/unidentified_caller", mt.Name, got) + } + for _, bad := range []string{"agent", "path", "model"} { + if _, ok := got[bad]; ok { + t.Errorf("%s carries forbidden attribute %q", mt.Name, bad) + } + } + } + } + if counts != 1 || durations != 1 { + t.Fatalf("counters = %d, histograms = %d, want 1 each", counts, durations) + } +} + +func attrKey(k string) attribute.Key { return attribute.Key(k) } diff --git a/src/crux/proxy/somark_linux.go b/src/crux/proxy/somark_linux.go deleted file mode 100644 index b7b0158b68..0000000000 --- a/src/crux/proxy/somark_linux.go +++ /dev/null @@ -1,20 +0,0 @@ -//go:build linux - -package proxy - -import ( - "syscall" -) - -// setSockMark sets SO_MARK on the raw socket file descriptor. The kernel then -// stamps every packet the socket emits with this fwmark, which the entrypoint's -// iptables nat/OUTPUT rule matches (-m mark --mark -j RETURN) to exempt -// the proxy's OWN upstream traffic from the :443 -> :18443 redirect, so it does -// not loop back into itself. -// -// Unlike `-m owner --uid-owner`, packet marks do NOT require the xt_owner kernel -// module, which is absent on OpenShift/OVN nodes. The mark exemption is the -// single self-exemption mechanism that works on both OKE and OpenShift. -func setSockMark(fd uintptr, mark int) error { - return syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_MARK, mark) -} diff --git a/src/crux/proxy/somark_other.go b/src/crux/proxy/somark_other.go deleted file mode 100644 index e5fee60dc7..0000000000 --- a/src/crux/proxy/somark_other.go +++ /dev/null @@ -1,11 +0,0 @@ -//go:build !linux - -package proxy - -// setSockMark is a no-op on non-Linux platforms: SO_MARK is a Linux-only socket -// option. The proxy only runs inside the Linux container, but the package must -// still compile on the CI build matrix (and on developer darwin machines), so -// this stub keeps the mark-setting dialer portable. -func setSockMark(fd uintptr, mark int) error { - return nil -} diff --git a/src/crux/proxy/upstream_chain_test.go b/src/crux/proxy/upstream_chain_test.go index 4c902d3483..9c68b4b5bf 100644 --- a/src/crux/proxy/upstream_chain_test.go +++ b/src/crux/proxy/upstream_chain_test.go @@ -23,6 +23,7 @@ type testConnectProxy struct { ln net.Listener targets atomic.Int32 last atomic.Value + refuse atomic.Bool } func startTestConnectProxy(t *testing.T) *testConnectProxy { @@ -55,6 +56,10 @@ func (cp *testConnectProxy) serve(c net.Conn) { } cp.targets.Add(1) cp.last.Store(req.Host) + if cp.refuse.Load() { + _, _ = io.WriteString(c, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") + return + } up, err := net.DialTimeout("tcp", req.Host, 5*time.Second) if err != nil { _, _ = io.WriteString(c, "HTTP/1.1 502 Bad Gateway\r\nContent-Length: 0\r\n\r\n") @@ -240,7 +245,8 @@ func TestDialUpstreamRawChainsAndGuardsPrivateLiterals(t *testing.T) { func TestDialUpstreamTLSSurfacesProxyRefusal(t *testing.T) { cp := startTestConnectProxy(t) p := newChainedTestProxy(t, cp, []string{"ghe.corp.invalid"}) - // The test proxy dials the target itself; an unresolvable name yields 502. + // The test proxy is told to refuse so the test does not depend on DNS. + cp.refuse.Store(true) _, err := p.dialUpstreamTLS("ghe.corp.invalid:443", "ghe.corp.invalid", 3*time.Second) if err == nil || !strings.Contains(err.Error(), "refused") { t.Fatalf("err = %v, want CONNECT refusal", err) diff --git a/src/crux/sandbox/openshell.go b/src/crux/sandbox/openshell.go index 76699f0d2f..20a4e8f521 100644 --- a/src/crux/sandbox/openshell.go +++ b/src/crux/sandbox/openshell.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "hash/fnv" "net/url" "os" "path/filepath" @@ -50,14 +51,25 @@ const ( // on the created sandbox. OpenShellPolicyProfileAnnotation = "hive.dev/policy-profile" - openshellNamePrefix = "hive-kick-" - openshellNameMax = 63 + // openshellNamePrefix and openshellNameMax follow the gateway's + // MAX_ROUTABLE_NAME_LEN (19): a sandbox name is one of three segments + // in a 63-char DNS label (----), so the + // prefix is short and the timestamp is base36 (measured on 0.1.8, + // hivecommons/hive#17643). + openshellNamePrefix = "hk-" + openshellNameMax = 19 + openshellNameUnixBase = 36 + openshellNameTagLen = 3 + openshellNameTagMod = 36 * 36 * 36 openshellRequestWait = 30 * time.Second openshellReadyPoll = 2 * time.Second openshellMaxOutput = 16 << 20 openshellManagedLabel = "app.kubernetes.io/managed-by" openshellManagedValue = "hive-openshell" - openshellHTTPSPort = 443 + // openshellWorkspaceVolume names hive's claim in driver_config; the + // gateway reserves the bare name "workspace" for its own volumes. + openshellWorkspaceVolume = "hive-workspace" + openshellHTTPSPort = 443 ) // OpenShellAlwaysAllowedHosts are the egress hosts every hive sandbox gets: @@ -366,12 +378,12 @@ func (l *OpenShellLauncher) execEnv(spec LaunchSpec) map[string]string { func (l *OpenShellLauncher) driverConfig(subPath, mount string, readOnly bool) map[string]any { k8s := map[string]any{ "volumes": []any{map[string]any{ - "name": "workspace", + "name": openshellWorkspaceVolume, "persistent_volume_claim": map[string]any{"claim_name": l.Options.WorkspaceClaim, "read_only": false}, }}, "containers": map[string]any{"agent": map[string]any{ "volume_mounts": []any{map[string]any{ - "name": "workspace", + "name": openshellWorkspaceVolume, "mount_path": mount, "sub_path": subPath, "read_only": readOnly, @@ -555,16 +567,28 @@ func openshellWorkspaceSubPath(workspace, claimMount string) (string, error) { var openshellNameUnsafe = regexp.MustCompile(`[^a-z0-9-]+`) // openshellSandboxName builds a unique, DNS-label-safe sandbox name from the -// launch name and a timestamp: hive-kick--. +// launch name and a timestamp: hk--, at most +// openshellNameMax characters. The launch name's hive-/-sandbox wrapping is +// dropped so the agent slug gets the room; a slug that still does not fit is +// cut and tagged with a short hash of the whole slug, so two agents sharing +// a long prefix never map to the same sandbox name. func openshellSandboxName(launchName string, now time.Time) string { - base := strings.Trim(openshellNameUnsafe.ReplaceAllString(strings.ToLower(strings.TrimSuffix(launchName, "-sandbox")), "-"), "-") + base := strings.ToLower(strings.TrimSuffix(launchName, launchNameSuffix)) + base = strings.TrimPrefix(base, launchNamePrefix) + base = strings.Trim(openshellNameUnsafe.ReplaceAllString(base, "-"), "-") if base == "" { base = "agent" } - suffix := fmt.Sprintf("-%d", now.Unix()) + suffix := "-" + strconv.FormatInt(now.Unix(), openshellNameUnixBase) room := openshellNameMax - len(openshellNamePrefix) - len(suffix) if len(base) > room { - base = strings.TrimRight(base[:room], "-") + h := fnv.New32a() + _, _ = h.Write([]byte(base)) + tag := strconv.FormatUint(uint64(h.Sum32())%openshellNameTagMod, openshellNameUnixBase) + for len(tag) < openshellNameTagLen { + tag = "0" + tag + } + base = strings.TrimRight(base[:room-openshellNameTagLen], "-") + tag } return openshellNamePrefix + base + suffix } diff --git a/src/crux/sandbox/openshell_logs.go b/src/crux/sandbox/openshell_logs.go index 4913ebb4ed..191c1a0a65 100644 --- a/src/crux/sandbox/openshell_logs.go +++ b/src/crux/sandbox/openshell_logs.go @@ -153,7 +153,7 @@ func openshellSandboxNameUnix(launchName, name string) (int64, bool) { if i < 0 || !strings.HasPrefix(name, openshellNamePrefix) { return 0, false } - unix, err := strconv.ParseInt(name[i+1:], 10, 64) + unix, err := strconv.ParseInt(name[i+1:], openshellNameUnixBase, 64) if err != nil { return 0, false } diff --git a/src/crux/sandbox/openshell_logs_test.go b/src/crux/sandbox/openshell_logs_test.go index 2da1939117..b688e9b9e1 100644 --- a/src/crux/sandbox/openshell_logs_test.go +++ b/src/crux/sandbox/openshell_logs_test.go @@ -270,7 +270,7 @@ func TestOpenShellSandboxNameUnix(t *testing.T) { {good, true}, {"nodash", false}, {"other-deps-1700000000", false}, - {openshellNamePrefix + "deps-notanumber", false}, + {openshellNamePrefix + "deps-not.a.number", false}, {openshellSandboxName(LaunchName("deps-bot"), time.Unix(1700000000, 0)), false}, } for _, c := range cases { diff --git a/src/crux/sandbox/openshell_test.go b/src/crux/sandbox/openshell_test.go index 6f20cfc2d6..864eb64cf9 100644 --- a/src/crux/sandbox/openshell_test.go +++ b/src/crux/sandbox/openshell_test.go @@ -145,7 +145,7 @@ func TestOpenShellRunSuccess(t *testing.T) { if c.workspace != OpenShellDefaultWorkspace { t.Errorf("workspace = %q", c.workspace) } - if c.name != "hive-kick-deps-1700000000" { + if c.name != "hk-deps-s44we8" { t.Errorf("name = %q", c.name) } if c.spec.Template.Image != "registry.example/dev:sha256-abc" { @@ -186,12 +186,17 @@ func TestOpenShellRunDriverConfigMountsWorkspaceClaim(t *testing.T) { t.Fatalf("driver config = %v", dc) } vols := k8s["volumes"].([]any) - pvc := vols[0].(map[string]any)["persistent_volume_claim"].(map[string]any) + vol := vols[0].(map[string]any) + // The gateway rejects a caller volume named "workspace" (reserved). + if vol["name"] != openshellWorkspaceVolume || vol["name"] == "workspace" { + t.Errorf("volume name = %v", vol["name"]) + } + pvc := vol["persistent_volume_claim"].(map[string]any) if pvc["claim_name"] != "hive-data" { t.Errorf("claim = %v", pvc) } mount := k8s["containers"].(map[string]any)["agent"].(map[string]any)["volume_mounts"].([]any)[0].(map[string]any) - if mount["sub_path"] != "agents/sandbox/deps" || mount["mount_path"] != OpenShellWorkspaceMount { + if mount["name"] != vol["name"] || mount["sub_path"] != "agents/sandbox/deps" || mount["mount_path"] != OpenShellWorkspaceMount { t.Errorf("mount = %v", mount) } pod := k8s["pod"].(map[string]any) @@ -362,16 +367,28 @@ func TestOpenShellAddress(t *testing.T) { func TestOpenShellSandboxName(t *testing.T) { now := time.Unix(1700000000, 0) - if got := openshellSandboxName("Deps Agent-sandbox", now); got != "hive-kick-deps-agent-1700000000" { + if got := openshellSandboxName("Deps Agent-sandbox", now); got != "hk-deps-aftv-s44we8" { t.Errorf("name = %q", got) } long := openshellSandboxName(strings.Repeat("a", 100), now) - if len(long) > openshellNameMax || !strings.HasSuffix(long, "-1700000000") { + // The gateway rejects routable names over 19 characters + // (MAX_ROUTABLE_NAME_LEN); the suffix is unix seconds in base36. + if len(long) != openshellNameMax || !strings.HasSuffix(long, "-s44we8") { t.Errorf("long name = %q (%d)", long, len(long)) } - if got := openshellSandboxName("---", now); got != "hive-kick-agent-1700000000" { + if got := openshellSandboxName("---", now); got != "hk-agent-s44we8" { t.Errorf("empty base = %q", got) } + // The launch name's hive-/-sandbox wrapping is dropped, and two agents + // that share a long prefix get distinct, stable names. + if got := openshellSandboxName(LaunchName("deps"), now); got != "hk-deps-s44we8" { + t.Errorf("launch name = %q", got) + } + one := openshellSandboxName(LaunchName("reviewer-one"), now) + two := openshellSandboxName(LaunchName("reviewer-two"), now) + if one == two || len(one) != openshellNameMax || one != openshellSandboxName(LaunchName("reviewer-one"), now) { + t.Errorf("truncated names = %q %q", one, two) + } } func TestOpenShellRunRefusesClineProvider(t *testing.T) { diff --git a/src/crux/sandbox/substrate.go b/src/crux/sandbox/substrate.go new file mode 100644 index 0000000000..965a3e8471 --- /dev/null +++ b/src/crux/sandbox/substrate.go @@ -0,0 +1,445 @@ +package sandbox + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "strings" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +// Agent Substrate runtime (ADR-0026 phase 2; sibling of the OpenShell runtime +// in openshell.go). +// +// SubstrateLauncher implements Launcher on top of a Substrate control plane +// (ate-api-server): it registers a per-kick ActorTemplate from the kick spec, +// creates an Actor from it, attaches the egress policy, resumes the actor, +// drives the kick through the in-actor bridge (substratebridge) over the +// atenet-router CONNECT tunnel, and deletes — or, with snapshot_on_idle, +// suspends — the actor afterwards. Substrate owns the gVisor/microVM sandbox, +// the snapshot store and the egress gateway; hive's part is the spec, the +// policy and the same cleanup and timeout semantics the Job launcher has. +// +// Two points differ from the OpenShell and Job runtimes and are deliberate: +// +// - Substrate has no exec/attach RPC and cannot mount an existing PVC, so +// the workspace is a per-actor durableDir and hive copies it in and out +// through the bridge (push, run, pull). The local workspace hive opened +// the kick in is therefore still the source of truth after the kick. +// - Substrate refuses tag-only images; the sandbox image must be pinned by +// digest (…@sha256:…). Validate enforces it up front so the failure is a +// config error, not a CreateActorTemplate rejection mid-kick. + +const ( + // SubstrateDefaultEndpoint is the in-cluster Control API address the + // default ate-setup install exposes. + SubstrateDefaultEndpoint = "api.ate-system.svc:443" + // SubstrateDefaultRouterURL is the in-cluster atenet-router CONNECT + // listener hive reaches actors through. + SubstrateDefaultRouterURL = "https://atenet-router.ate-system.svc:8444" + // SubstrateDefaultAtespace is the atespace actors land in when none is + // configured. + SubstrateDefaultAtespace = "hive" + // SubstrateDefaultTokenFile is the projected ServiceAccount token + // (audience api.ate-system.svc) the Control API authenticates. + SubstrateDefaultTokenFile = "/var/run/secrets/ate/token" + // SubstrateDefaultCAFile is the servicedns ClusterTrustBundle projection + // the Control API serving certificate chains to. + SubstrateDefaultCAFile = "/var/run/secrets/ate/ca.crt" + // SubstrateDefaultRouterClientBundle is the projected podCertificate + // bundle atenet-router authenticates hive with. + SubstrateDefaultRouterClientBundle = "/run/router-client/credential-bundle.pem" + // SubstrateDefaultRouterCAFile is the servicedns ClusterTrustBundle the + // router's serving certificate chains to. + SubstrateDefaultRouterCAFile = "/run/router-client/router-ca.pem" + // SubstrateWorkspaceMount is where the actor sees the per-kick + // workspace, matching the other launchers. + SubstrateWorkspaceMount = substratebridge.DefaultWorkspace + // SubstrateDefaultCPUMillicores and SubstrateDefaultMemoryBytes are the + // template resource limits when the operator sets none; Substrate + // requires both. + SubstrateDefaultCPUMillicores = 2000 + SubstrateDefaultMemoryBytes = 4 << 30 + // SubstrateExitCodeUnknown mirrors OpenShellExitCodeUnknown. + SubstrateExitCodeUnknown = -1 + + // SubstratePolicyProfileLabel carries sandbox.substrate.policy_profile on + // the per-kick template's worker selector key space; Substrate has no + // annotations, so the profile travels as a template label hive and + // operators can filter on. + SubstratePolicyProfileLabel = "hive.dev/policy-profile" + + substrateNamePrefix = "hive-kick-" + substrateNameMax = 63 + substrateRequestWait = 30 * time.Second + substrateReadyPoll = 2 * time.Second + substrateEgressPolicy = "default" + substrateContainer = "agent" + substrateVolume = "workspace" + substrateHTTPSPort = 443 +) + +// SubstrateOptions is the launcher's configuration; crux/agent builds it +// from config.SandboxSubstrateConfig. +type SubstrateOptions struct { + Endpoint string + Atespace string + Token string + TokenFile string + CAFile string + InsecureTLS bool + Timeout time.Duration + RouterURL string + RouterClientBundle string + RouterCAFile string + BridgePort int + Image string + Command []string + CPUMillicores int64 + MemoryBytes int64 + WorkerSelector map[string]string + SnapshotOnIdle bool + AllowHosts []string + PolicyProfile string + // Logger receives cleanup diagnostics that do not fail the kick; nil + // means slog.Default(). + Logger *slog.Logger +} + +func (o SubstrateOptions) logger() *slog.Logger { + if o.Logger != nil { + return o.Logger + } + return slog.Default() +} + +// Validate reports the first configuration error a kick would hit. +func (o SubstrateOptions) Validate() error { + if _, _, err := substrateAddress(o.endpoint()); err != nil { + return err + } + if _, err := substrateRouterAddress(o.routerURL()); err != nil { + return err + } + if o.Image != "" && !substrateImageDigest.MatchString(o.Image) { + return fmt.Errorf("substrate: image %q must be pinned by digest (…@sha256:<64 hex>); Substrate refuses tag-only images", o.Image) + } + if o.CPUMillicores < 0 || o.MemoryBytes < 0 { + return errors.New("substrate: cpu_millicores and memory_bytes must not be negative") + } + if o.BridgePort < 0 || o.BridgePort > 65535 { + return fmt.Errorf("substrate: bridge_port %d is out of range", o.BridgePort) + } + _, err := SubstrateEgressPolicy(o.AllowHosts) + return err +} + +func (o SubstrateOptions) endpoint() string { + if e := strings.TrimSpace(o.Endpoint); e != "" { + return e + } + return SubstrateDefaultEndpoint +} + +func (o SubstrateOptions) atespace() string { + if a := strings.TrimSpace(o.Atespace); a != "" { + return a + } + return SubstrateDefaultAtespace +} + +func (o SubstrateOptions) routerURL() string { + if r := strings.TrimSpace(o.RouterURL); r != "" { + return r + } + return SubstrateDefaultRouterURL +} + +func (o SubstrateOptions) bridgePort() int { + if o.BridgePort > 0 { + return o.BridgePort + } + return substratebridge.DefaultPort +} + +// substrateControl is the slice of the Control API the launcher uses; it +// exists so tests can drive the launcher with a fake and so an upstream +// RPC rename shows up here, not in every call site. +type substrateControl interface { + CreateActorTemplate(ctx context.Context, tpl *substrateapi.ActorTemplate) (*substrateapi.ActorTemplate, error) + DeleteActorTemplate(ctx context.Context, ref *substrateapi.ObjectRef) error + CreateActor(ctx context.Context, actor *substrateapi.Actor) (*substrateapi.Actor, error) + GetActor(ctx context.Context, ref *substrateapi.ObjectRef) (*substrateapi.Actor, error) + ResumeActor(ctx context.Context, ref *substrateapi.ObjectRef) error + SuspendActor(ctx context.Context, ref *substrateapi.ObjectRef, mode substrateapi.DurabilityMode) error + DeleteActor(ctx context.Context, ref *substrateapi.ObjectRef) error + CreateActorEgressPolicy(ctx context.Context, ref *substrateapi.ObjectRef, policy *substrateapi.EgressPolicy) error + ListWorkers(ctx context.Context) ([]*substrateapi.Worker, error) +} + +// substrateBridge is the launcher's view of the in-actor bridge. +type substrateBridge interface { + WaitReady(ctx context.Context, ref *substrateapi.ObjectRef, poll time.Duration) error + Push(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error + // Run copies the kick's output to live (nil allowed) as it arrives. + Run(ctx context.Context, ref *substrateapi.ObjectRef, req substratebridge.RunRequest, live io.Writer) (substratebridge.RunResponse, error) + Pull(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error +} + +type substrateClient interface { + Control() substrateControl + Bridge() substrateBridge + Close() error +} + +// SubstrateLauncher runs sandboxed kicks as Substrate actors. +type SubstrateLauncher struct { + Options SubstrateOptions + + // newClient overrides the client constructor in tests. + newClient func(SubstrateOptions) (substrateClient, error) + // now overrides the clock in tests. + now func() time.Time + // kicks is where each kick's actor states and output are recorded for + // the dashboard's terminal pane; nil means DefaultSubstrateKicks. + kicks *SubstrateKicks +} + +func (l *SubstrateLauncher) kickRecord() *SubstrateKicks { + if l.kicks != nil { + return l.kicks + } + return DefaultSubstrateKicks +} + +func (l *SubstrateLauncher) client() (substrateClient, error) { + if l.newClient != nil { + return l.newClient(l.Options) + } + return newSubstrateGRPCClient(l.Options) +} + +// Run implements Launcher. +func (l *SubstrateLauncher) Run(ctx context.Context, spec LaunchSpec) (Result, error) { + res := Result{ExitCode: SubstrateExitCodeUnknown} + if spec.ClineProvider != nil { + _ = spec.ClineProvider.Close() + return res, errors.New("substrate: scoped Cline provider files are Podman-only; the Substrate runtime takes no credential files") + } + if err := l.Options.Validate(); err != nil { + return res, err + } + // Validate just proved the hosts parse, so this cannot fail; the policy + // is built before anything is created so a kick never has a half-built + // actor to clean up on a config error. + policy, err := SubstrateEgressPolicy(l.Options.AllowHosts) + if err != nil { + return res, err + } + image := strings.TrimSpace(l.Options.Image) + if image == "" { + image = strings.TrimSpace(spec.Image) + } + if image == "" { + return res, errors.New("substrate: image is required") + } + if !substrateImageDigest.MatchString(image) { + return res, fmt.Errorf("substrate: image %q must be pinned by digest (…@sha256:<64 hex>); set sandbox.substrate.image to the digest-pinned sandbox image", image) + } + if strings.TrimSpace(spec.Workspace) == "" { + return res, errors.New("substrate: workspace is required") + } + if len(spec.Command) == 0 { + return res, errors.New("substrate: command is required") + } + if l.Options.Timeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, l.Options.Timeout) + defer cancel() + } + + client, err := l.client() + if err != nil { + return res, err + } + defer func() { _ = client.Close() }() + control, bridge := client.Control(), client.Bridge() + + now := time.Now + if l.now != nil { + now = l.now + } + name := substrateActorName(spec.Name, now()) + ref := &substrateapi.ObjectRef{Atespace: l.Options.atespace(), Name: name} + + // The caller's ctx may be dead already (kick timeout) or die between any + // two of these requests; cleanup falls back to a fresh, request-bounded + // context so a finished actor is still removed. + var cancels []context.CancelFunc + defer func() { + for _, c := range cancels { + c() + } + }() + live := func() context.Context { + if ctx.Err() == nil { + return ctx + } + fresh, cancel := context.WithTimeout(context.Background(), substrateRequestWait) + cancels = append(cancels, cancel) + return fresh + } + + if _, err := control.CreateActorTemplate(ctx, l.actorTemplate(ref, image)); err != nil { + return res, fmt.Errorf("substrate: creating actor template %s: %w", name, err) + } + cleanupTemplate := func() { _ = control.DeleteActorTemplate(live(), ref) } + + created, err := control.CreateActor(ctx, &substrateapi.Actor{ + Metadata: &substrateapi.ResourceMetadata{Atespace: ref.Atespace, Name: name}, + ActorTemplate: ref, + WorkerSelector: substrateSelector(l.Options.WorkerSelector), + }) + if err != nil { + cleanupTemplate() + return res, fmt.Errorf("substrate: creating actor %s: %w", name, err) + } + kicks := l.kickRecord() + kick := kicks.begin(spec.Name, SubstrateActorInfo{Name: name, Atespace: ref.Atespace, State: SubstrateStateCreated}) + kicks.observe(kick, created) + // Delete is the default: a kick is a one-shot and keeping a snapshot + // around costs object storage. snapshot_on_idle keeps the actor + // suspended instead. Nothing resumes it yet (the next kick is a cold + // boot), so only the latest snapshot per agent is retained: the one it + // replaces is deleted. The Control API has no ListActors, so actors left + // by an earlier hive process are not reclaimed. + cleanupActor := func() { + if l.Options.SnapshotOnIdle && ctx.Err() == nil { + if err := control.SuspendActor(live(), ref, substrateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS); err == nil { + kicks.update(kick, func(i *SubstrateActorInfo) { + i.State, i.Suspended, i.Done = SubstrateStateSuspended, true, true + }) + cleanupTemplate() + prev := kicks.swapSuspended(spec.Name, SubstrateActorInfo{Name: name, Atespace: ref.Atespace}) + switch { + case prev.Name == "" || prev.Name == name: + case prev.Atespace != ref.Atespace: + // This launcher's client is for the current atespace; + // an actor left in another one is not ours to reach. + l.Options.logger().Warn("substrate: superseded suspended actor left in another atespace", + "agent", spec.Name, "actor", prev.Name, "atespace", prev.Atespace, "current_atespace", ref.Atespace) + default: + prevRef := &substrateapi.ObjectRef{Atespace: prev.Atespace, Name: prev.Name} + if err := control.DeleteActor(live(), prevRef); err != nil { + l.Options.logger().Warn("substrate: deleting superseded suspended actor", + "agent", spec.Name, "actor", prev.Name, "atespace", prev.Atespace, "error", err) + } + } + return + } + } + _ = control.DeleteActor(live(), ref) + kicks.update(kick, func(i *SubstrateActorInfo) { i.State, i.Done = SubstrateStateDeleting, true }) + cleanupTemplate() + } + + if spec.NetworkNone { + policy.Rules = nil + } + // The Control API requires the singleton policy's atespace to match the + // actor's (egress_policy.metadata.atespace is a required field). + policy.Metadata.Atespace = ref.Atespace + if err := control.CreateActorEgressPolicy(ctx, ref, policy); err != nil { + cleanupActor() + return res, fmt.Errorf("substrate: setting egress policy on actor %s: %w", name, err) + } + if err := control.ResumeActor(ctx, ref); err != nil { + cleanupActor() + return res, fmt.Errorf("substrate: resuming actor %s: %w", name, err) + } + kicks.update(kick, func(i *SubstrateActorInfo) { i.State = SubstrateStateResuming }) + observe := func(a *substrateapi.Actor) { kicks.observe(kick, a) } + if err := substrateWaitRunning(ctx, control, ref, substrateReadyPoll, observe); err != nil { + cleanupActor() + if ctx.Err() != nil { + return res, ctx.Err() + } + return res, fmt.Errorf("substrate: actor %s never became running: %w", name, err) + } + if err := bridge.WaitReady(ctx, ref, substrateReadyPoll); err != nil { + cleanupActor() + if ctx.Err() != nil { + return res, ctx.Err() + } + return res, fmt.Errorf("substrate: bridge in actor %s never answered: %w", name, err) + } + if err := bridge.Push(ctx, ref, spec.Workspace); err != nil { + cleanupActor() + return res, fmt.Errorf("substrate: pushing workspace into actor %s: %w", name, err) + } + workDir := spec.WorkDir + if workDir == "" { + workDir = SubstrateWorkspaceMount + } + out, runErr := bridge.Run(ctx, ref, substratebridge.RunRequest{Command: spec.Command, Env: substrateEnv(spec), WorkDir: workDir}, kicks.output(kick)) + if runErr == nil { + res.Stdout, res.Stderr, res.ExitCode = out.Stdout, out.Stderr, out.ExitCode + kicks.update(kick, func(i *SubstrateActorInfo) { i.Exited, i.ExitCode = true, out.ExitCode }) + if out.Error != "" { + runErr = errors.New(out.Error) + } + } + var pullErr error + if ctx.Err() == nil { + pullErr = bridge.Pull(ctx, ref, spec.Workspace) + } + cleanupActor() + if ctx.Err() != nil { + // Kick timeout or cancel: the actor has been removed above so the + // workload does not keep running on the cluster. + return res, ctx.Err() + } + if runErr != nil { + return res, fmt.Errorf("substrate: running kick in actor %s: %w", name, runErr) + } + if pullErr != nil { + return res, fmt.Errorf("substrate: pulling workspace out of actor %s: %w", name, pullErr) + } + return res, nil +} + +// substrateWaitRunning polls GetActor until the actor is RUNNING, or fails +// when it crashes. A deleted or suspended actor during the wait is a failure +// too: nothing hive did asked for it. observe (nil allowed) sees every +// actor GetActor returns. +func substrateWaitRunning(ctx context.Context, control substrateControl, ref *substrateapi.ObjectRef, poll time.Duration, observe func(*substrateapi.Actor)) error { + ticker := time.NewTicker(poll) + defer ticker.Stop() + for { + actor, err := control.GetActor(ctx, ref) + if err != nil { + return err + } + if observe != nil { + observe(actor) + } + switch state := actor.GetStatus().GetState(); state { + case substrateapi.ActorState_ACTOR_STATE_RUNNING: + return nil + case substrateapi.ActorState_ACTOR_STATE_CRASHED: + return fmt.Errorf("actor crashed: %s", actor.GetStatus().GetCrash().GetMessage()) + case substrateapi.ActorState_ACTOR_STATE_DELETING, substrateapi.ActorState_ACTOR_STATE_SUSPENDING: + return fmt.Errorf("actor entered %s while being resumed", state) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-ticker.C: + } + } +} diff --git a/src/crux/sandbox/substrate_bridge.go b/src/crux/sandbox/substrate_bridge.go new file mode 100644 index 0000000000..5762b39bc8 --- /dev/null +++ b/src/crux/sandbox/substrate_bridge.go @@ -0,0 +1,147 @@ +package sandbox + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +// routerBridge's substrateBridge implementation: the four verbs the launcher +// drives a kick with, each one HTTP exchange over its own CONNECT tunnel. + +// WaitReady polls the bridge's health endpoint until it answers. The actor +// is RUNNING by now, but the bridge process inside still has to come up. +func (b *routerBridge) WaitReady(ctx context.Context, ref *substrateapi.ObjectRef, poll time.Duration) error { + ticker := time.NewTicker(poll) + defer ticker.Stop() + var last error + for { + resp, err := b.do(ctx, ref, http.MethodGet, substratebridge.PathHealth, nil) + if err == nil { + resp.Body.Close() + return nil + } + last = err + select { + case <-ctx.Done(): + return fmt.Errorf("%w (last: %v)", ctx.Err(), last) + case <-ticker.C: + } + } +} + +// Push replaces the actor's workspace with the local one. +func (b *routerBridge) Push(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error { + var buf bytes.Buffer + if err := substratebridge.Pack(workspace, &buf); err != nil { + return fmt.Errorf("packing %s: %w", workspace, err) + } + resp, err := b.do(ctx, ref, http.MethodPut, substratebridge.PathWorkspace, &buf) + if err != nil { + return err + } + return resp.Body.Close() +} + +// Run executes the kick command in the actor and waits for it to exit. It +// asks the bridge to stream (RunRequest.Stream) and copies stdout and +// stderr to live as they arrive, so the dashboard's terminal pane can follow +// the kick; the returned RunResponse still carries the whole (capped) +// output for the transcript. live may be nil. +func (b *routerBridge) Run(ctx context.Context, ref *substrateapi.ObjectRef, req substratebridge.RunRequest, live io.Writer) (substratebridge.RunResponse, error) { + var out substratebridge.RunResponse + req.Stream = true + body, err := json.Marshal(req) + if err != nil { + return out, err + } + resp, err := b.do(ctx, ref, http.MethodPost, substratebridge.PathRun, bytes.NewReader(body)) + if err != nil { + return out, err + } + defer resp.Body.Close() + return substrateDecodeRun(resp.Body, live) +} + +// substrateDecodeRun reads a /run answer: Partial frames from a streaming +// bridge, then the final frame with the exit code. A bridge that does not +// stream answers with one complete, non-Partial RunResponse, which is read +// the same way. +func substrateDecodeRun(r io.Reader, live io.Writer) (substratebridge.RunResponse, error) { + var out substratebridge.RunResponse + var stdout, stderr bytes.Buffer + dec := json.NewDecoder(r) + for { + var frame substratebridge.RunResponse + if err := dec.Decode(&frame); err != nil { + if errors.Is(err, io.EOF) { + err = io.ErrUnexpectedEOF + } + return out, fmt.Errorf("decoding run response: %w", err) + } + substrateAppendCapped(&stdout, frame.Stdout) + substrateAppendCapped(&stderr, frame.Stderr) + if live != nil { + _, _ = io.WriteString(live, frame.Stdout) + _, _ = io.WriteString(live, frame.Stderr) + } + if !frame.Partial { + out.Stdout, out.Stderr = stdout.String(), stderr.String() + out.ExitCode, out.Error = frame.ExitCode, frame.Error + return out, nil + } + } +} + +func substrateAppendCapped(buf *bytes.Buffer, s string) { + if room := substratebridge.MaxOutput - buf.Len(); len(s) > room { + s = s[:max(room, 0)] + } + buf.WriteString(s) +} + +// Pull replaces the local workspace with the actor's, so the commits the +// agent made are what hive's post-kick push broker sees. The new tree is +// unpacked beside the old one and swapped in, so a failed transfer leaves +// the original workspace untouched. +func (b *routerBridge) Pull(ctx context.Context, ref *substrateapi.ObjectRef, workspace string) error { + resp, err := b.do(ctx, ref, http.MethodGet, substratebridge.PathWorkspace, nil) + if err != nil { + return err + } + defer resp.Body.Close() + return substrateReplaceDir(resp.Body, workspace) +} + +func substrateReplaceDir(stream io.Reader, workspace string) error { + parent := filepath.Dir(filepath.Clean(workspace)) + staging, err := os.MkdirTemp(parent, ".substrate-pull-*") + if err != nil { + return err + } + if err := substratebridge.Unpack(stream, staging); err != nil { + _ = os.RemoveAll(staging) + return fmt.Errorf("unpacking into %s: %w", staging, err) + } + old := staging + ".old" + if err := os.Rename(workspace, old); err != nil { + _ = os.RemoveAll(staging) + return err + } + if err := os.Rename(staging, workspace); err != nil { + _ = os.Rename(old, workspace) + _ = os.RemoveAll(staging) + return err + } + return os.RemoveAll(old) +} diff --git a/src/crux/sandbox/substrate_client.go b/src/crux/sandbox/substrate_client.go new file mode 100644 index 0000000000..2e72b7e4a5 --- /dev/null +++ b/src/crux/sandbox/substrate_client.go @@ -0,0 +1,292 @@ +package sandbox + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strings" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" +) + +// Real Substrate client: the Control API over gRPC plus the in-actor bridge +// over an atenet-router CONNECT tunnel. Constructed lazily by +// SubstrateLauncher.client() so unit tests never dial anything. + +// substrateRouterHeader selects the target actor on the router's CONNECT +// request (api-guide "Workload Connectivity"). +const substrateRouterHeader = "ate-target-actor" + +type grpcSubstrateClient struct { + conn *grpc.ClientConn + control *grpcControl + bridge *routerBridge +} + +func (c *grpcSubstrateClient) Control() substrateControl { return c.control } +func (c *grpcSubstrateClient) Bridge() substrateBridge { return c.bridge } +func (c *grpcSubstrateClient) Close() error { return c.conn.Close() } + +func newSubstrateGRPCClient(o SubstrateOptions) (substrateClient, error) { + addr, useTLS, err := substrateAddress(o.endpoint()) + if err != nil { + return nil, err + } + creds := insecure.NewCredentials() + if useTLS { + cfg, err := substrateControlTLS(o) + if err != nil { + return nil, err + } + creds = credentials.NewTLS(cfg) + } + opts := []grpc.DialOption{grpc.WithTransportCredentials(creds)} + if o.Token != "" || o.TokenFile != "" || useTLS { + opts = append(opts, grpc.WithPerRPCCredentials(substrateBearer{token: o.Token, file: o.tokenFile(), secure: useTLS})) + } + conn, err := grpc.NewClient(addr, opts...) + if err != nil { + return nil, fmt.Errorf("substrate: connecting to control api %s: %w", addr, err) + } + bridge, err := newRouterBridge(o) + if err != nil { + _ = conn.Close() + return nil, err + } + return &grpcSubstrateClient{conn: conn, control: &grpcControl{c: substrateapi.NewControlClient(conn)}, bridge: bridge}, nil +} + +func (o SubstrateOptions) tokenFile() string { + if f := strings.TrimSpace(o.TokenFile); f != "" { + return f + } + return SubstrateDefaultTokenFile +} + +// substrateControlTLS verifies the Control API against ca_file (the +// servicedns ClusterTrustBundle projection) when present, the system roots +// otherwise, or nothing with insecure_tls. +func substrateControlTLS(o SubstrateOptions) (*tls.Config, error) { + cfg := &tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: o.InsecureTLS} //nolint:gosec // operator opt-in + if o.InsecureTLS { + return cfg, nil + } + caFile := strings.TrimSpace(o.CAFile) + if caFile == "" { + if _, err := os.Stat(SubstrateDefaultCAFile); err == nil { + caFile = SubstrateDefaultCAFile + } + } + if caFile != "" { + pool, err := substrateCertPool(caFile) + if err != nil { + return nil, fmt.Errorf("substrate: ca_file: %w", err) + } + cfg.RootCAs = pool + } + return cfg, nil +} + +func substrateCertPool(path string) (*x509.CertPool, error) { + pem, err := os.ReadFile(path) + if err != nil { + return nil, err + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, fmt.Errorf("%s holds no PEM certificates", path) + } + return pool, nil +} + +// substrateBearer sends the projected ServiceAccount token as a bearer +// token, re-reading the file per RPC so kubelet rotation is picked up. +type substrateBearer struct { + token string + file string + secure bool +} + +func (b substrateBearer) GetRequestMetadata(context.Context, ...string) (map[string]string, error) { + tok := b.token + if tok == "" { + raw, err := os.ReadFile(b.file) + if err != nil { + return nil, fmt.Errorf("substrate: reading token file %s: %w", b.file, err) + } + tok = strings.TrimSpace(string(raw)) + } + if tok == "" { + return nil, errors.New("substrate: control api token is empty") + } + return map[string]string{"authorization": "Bearer " + tok}, nil +} + +func (b substrateBearer) RequireTransportSecurity() bool { return b.secure } + +// grpcControl adapts the generated ControlClient to substrateControl. +type grpcControl struct{ c substrateapi.ControlClient } + +func (g *grpcControl) CreateActorTemplate(ctx context.Context, tpl *substrateapi.ActorTemplate) (*substrateapi.ActorTemplate, error) { + return g.c.CreateActorTemplate(ctx, &substrateapi.CreateActorTemplateRequest{ActorTemplate: tpl}) +} + +func (g *grpcControl) DeleteActorTemplate(ctx context.Context, ref *substrateapi.ObjectRef) error { + _, err := g.c.DeleteActorTemplate(ctx, &substrateapi.DeleteActorTemplateRequest{ActorTemplate: ref}) + return err +} + +func (g *grpcControl) CreateActor(ctx context.Context, actor *substrateapi.Actor) (*substrateapi.Actor, error) { + return g.c.CreateActor(ctx, &substrateapi.CreateActorRequest{Actor: actor}) +} + +func (g *grpcControl) GetActor(ctx context.Context, ref *substrateapi.ObjectRef) (*substrateapi.Actor, error) { + return g.c.GetActor(ctx, &substrateapi.GetActorRequest{Actor: ref}) +} + +func (g *grpcControl) ResumeActor(ctx context.Context, ref *substrateapi.ObjectRef) error { + _, err := g.c.ResumeActor(ctx, &substrateapi.ResumeActorRequest{Actor: ref}) + return err +} + +func (g *grpcControl) SuspendActor(ctx context.Context, ref *substrateapi.ObjectRef, mode substrateapi.DurabilityMode) error { + _, err := g.c.SuspendActor(ctx, &substrateapi.SuspendActorRequest{Actor: ref, Options: &substrateapi.SuspendOptions{Mode: mode}}) + return err +} + +func (g *grpcControl) DeleteActor(ctx context.Context, ref *substrateapi.ObjectRef) error { + _, err := g.c.DeleteActor(ctx, &substrateapi.DeleteActorRequest{Actor: ref}) + return err +} + +func (g *grpcControl) CreateActorEgressPolicy(ctx context.Context, ref *substrateapi.ObjectRef, policy *substrateapi.EgressPolicy) error { + _, err := g.c.CreateActorEgressPolicy(ctx, &substrateapi.CreateActorEgressPolicyRequest{Actor: ref, EgressPolicy: policy}) + return err +} + +func (g *grpcControl) ListWorkers(ctx context.Context) ([]*substrateapi.Worker, error) { + var out []*substrateapi.Worker + req := &substrateapi.ListWorkersRequest{Options: &substrateapi.ListOptions{}} + for { + resp, err := g.c.ListWorkers(ctx, req) + if err != nil { + return nil, err + } + out = append(out, resp.GetWorkers()...) + if resp.GetNextPageToken() == "" { + return out, nil + } + req.Options.PageToken = resp.GetNextPageToken() + } +} + +// routerBridge reaches the in-actor bridge through atenet-router: an mTLS +// CONNECT to the router with the actor named in ate-target-actor, then plain +// HTTP to the bridge over the tunnel. +type routerBridge struct { + routerAddr string + tlsConfig *tls.Config + port int + dial func(ctx context.Context, network, addr string) (net.Conn, error) +} + +func newRouterBridge(o SubstrateOptions) (*routerBridge, error) { + addr, err := substrateRouterAddress(o.routerURL()) + if err != nil { + return nil, err + } + host, _, _ := net.SplitHostPort(addr) + cfg := &tls.Config{MinVersion: tls.VersionTLS12, ServerName: host, InsecureSkipVerify: o.InsecureTLS} //nolint:gosec // operator opt-in + bundle := o.RouterClientBundle + if bundle == "" { + bundle = SubstrateDefaultRouterClientBundle + } + cert, err := tls.LoadX509KeyPair(bundle, bundle) + if err != nil { + return nil, fmt.Errorf("substrate: router_client_bundle %s: %w — atenet-router needs a podCertificate projection for hive (see deploy/substrate.md)", bundle, err) + } + cfg.Certificates = []tls.Certificate{cert} + if !o.InsecureTLS { + caFile := o.RouterCAFile + if caFile == "" { + caFile = SubstrateDefaultRouterCAFile + } + pool, err := substrateCertPool(caFile) + if err != nil { + return nil, fmt.Errorf("substrate: router_ca_file: %w", err) + } + cfg.RootCAs = pool + } + d := &net.Dialer{Timeout: substrateRequestWait} + return &routerBridge{routerAddr: addr, tlsConfig: cfg, port: o.bridgePort(), dial: d.DialContext}, nil +} + +// tunnel opens one CONNECT tunnel to the actor's bridge port. +func (b *routerBridge) tunnel(ctx context.Context, ref *substrateapi.ObjectRef) (net.Conn, error) { + raw, err := b.dial(ctx, "tcp", b.routerAddr) + if err != nil { + return nil, err + } + conn := tls.Client(raw, b.tlsConfig) + if err := conn.HandshakeContext(ctx); err != nil { + raw.Close() + return nil, fmt.Errorf("router tls handshake: %w", err) + } + target := net.JoinHostPort("actor", fmt.Sprint(b.port)) + req := &http.Request{Method: http.MethodConnect, URL: &url.URL{Opaque: target}, Host: target, Header: http.Header{}} + req.Header.Set(substrateRouterHeader, ref.GetAtespace()+"/"+ref.GetName()) + if deadline, ok := ctx.Deadline(); ok { + _ = conn.SetDeadline(deadline) + } + if err := req.Write(conn); err != nil { + conn.Close() + return nil, fmt.Errorf("router connect: %w", err) + } + resp, err := http.ReadResponse(bufio.NewReader(conn), req) + if err != nil { + conn.Close() + return nil, fmt.Errorf("router connect: %w", err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusOK { + conn.Close() + return nil, fmt.Errorf("router refused connect: %s", resp.Status) + } + _ = conn.SetDeadline(time.Time{}) + return conn, nil +} + +func (b *routerBridge) do(ctx context.Context, ref *substrateapi.ObjectRef, method, path string, body io.Reader) (*http.Response, error) { + transport := &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { return b.tunnel(ctx, ref) }, + DisableKeepAlives: true, + } + defer transport.CloseIdleConnections() + req, err := http.NewRequestWithContext(ctx, method, "http://actor"+path, body) + if err != nil { + return nil, err + } + resp, err := (&http.Client{Transport: transport}).Do(req) + if err != nil { + return nil, err + } + if resp.StatusCode/100 != 2 { + msg, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + resp.Body.Close() + return nil, fmt.Errorf("bridge %s %s: %s: %s", method, path, resp.Status, strings.TrimSpace(string(msg))) + } + return resp, nil +} diff --git a/src/crux/sandbox/substrate_client_test.go b/src/crux/sandbox/substrate_client_test.go new file mode 100644 index 0000000000..5a5638de27 --- /dev/null +++ b/src/crux/sandbox/substrate_client_test.go @@ -0,0 +1,685 @@ +package sandbox + +import ( + "bufio" + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "errors" + "io" + "log" + "math/big" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +// Real-client tests: the gRPC adapter against an in-memory Control server, +// TLS/credential plumbing against files in a temp dir, and the router bridge +// against a TLS server that speaks CONNECT and then the bridge protocol. + +// selfSignedBundle writes a PEM file holding a self-signed cert and its key +// (the shape of a podCertificate credential bundle) plus a CA-only file. +func selfSignedBundle(t *testing.T, dir string) (bundle, caOnly string, cert tls.Certificate) { + t.Helper() + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + t.Fatal(err) + } + tpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "hive-test"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth, x509.ExtKeyUsageClientAuth}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + DNSNames: []string{"localhost"}, + IsCA: true, + BasicConstraintsValid: true, + } + der, err := x509.CreateCertificate(rand.Reader, tpl, tpl, &key.PublicKey, key) + if err != nil { + t.Fatal(err) + } + keyDER, err := x509.MarshalECPrivateKey(key) + if err != nil { + t.Fatal(err) + } + certPEM := pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: der}) + keyPEM := pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) + bundle = filepath.Join(dir, "credential-bundle.pem") + caOnly = filepath.Join(dir, "router-ca.pem") + if err := os.WriteFile(bundle, append(certPEM, keyPEM...), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(caOnly, certPEM, 0o600); err != nil { + t.Fatal(err) + } + cert, err = tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + t.Fatal(err) + } + return bundle, caOnly, cert +} + +func TestNewSubstrateGRPCClientConstructs(t *testing.T) { + dir := t.TempDir() + bundle, ca, _ := selfSignedBundle(t, dir) + + // TLS endpoint verified against ca_file, router verified against its CA. + c, err := newSubstrateGRPCClient(SubstrateOptions{ + Endpoint: "api.ate-system.svc:443", CAFile: ca, TokenFile: filepath.Join(dir, "token"), + RouterClientBundle: bundle, RouterCAFile: ca, + }) + if err != nil { + t.Fatalf("tls client: %v", err) + } + if c.Control() == nil || c.Bridge() == nil { + t.Error("nil control or bridge") + } + if err := c.Close(); err != nil { + t.Errorf("Close: %v", err) + } + + // Plaintext endpoint with an env token and insecure router. + c, err = newSubstrateGRPCClient(SubstrateOptions{Endpoint: "http://localhost:8080", Token: "t", InsecureTLS: true, RouterClientBundle: bundle}) + if err != nil { + t.Fatalf("plaintext client: %v", err) + } + _ = c.Close() + + // Plaintext without any token still constructs (no per-RPC creds). + c, err = newSubstrateGRPCClient(SubstrateOptions{Endpoint: "http://localhost:8080", InsecureTLS: true, RouterClientBundle: bundle}) + if err != nil { + t.Fatalf("plaintext no-token client: %v", err) + } + _ = c.Close() + + // Insecure TLS skips the CA file entirely. + c, err = newSubstrateGRPCClient(SubstrateOptions{InsecureTLS: true, CAFile: "/nonexistent", RouterClientBundle: bundle}) + if err != nil { + t.Fatalf("insecure client: %v", err) + } + _ = c.Close() +} + +func TestNewSubstrateGRPCClientErrors(t *testing.T) { + dir := t.TempDir() + bundle, ca, _ := selfSignedBundle(t, dir) + noPEM := filepath.Join(dir, "empty.pem") + if err := os.WriteFile(noPEM, []byte("not a cert"), 0o600); err != nil { + t.Fatal(err) + } + cases := []struct { + name string + o SubstrateOptions + want string + }{ + {"endpoint", SubstrateOptions{Endpoint: "nope"}, "endpoint"}, + {"ca missing", SubstrateOptions{CAFile: "/nonexistent/ca.crt"}, "ca_file"}, + {"ca not pem", SubstrateOptions{CAFile: noPEM}, "no PEM certificates"}, + {"router url", SubstrateOptions{CAFile: ca, RouterURL: "http://r"}, "router_url"}, + {"router bundle", SubstrateOptions{CAFile: ca, RouterClientBundle: "/nonexistent/b.pem"}, "router_client_bundle"}, + {"router ca", SubstrateOptions{CAFile: ca, RouterClientBundle: bundle, RouterCAFile: "/nonexistent/ca.pem"}, "router_ca_file"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := newSubstrateGRPCClient(tc.o) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want %q", err, tc.want) + } + }) + } +} + +func TestSubstrateControlTLSDefaults(t *testing.T) { + cfg, err := substrateControlTLS(SubstrateOptions{}) + if err != nil || cfg.RootCAs != nil || cfg.InsecureSkipVerify { + t.Errorf("default tls = %+v, %v (system roots expected when the projected CA is absent)", cfg, err) + } + _, ca, _ := selfSignedBundle(t, t.TempDir()) + cfg, err = substrateControlTLS(SubstrateOptions{CAFile: ca}) + if err != nil || cfg.RootCAs == nil { + t.Errorf("ca_file tls = %+v, %v", cfg, err) + } +} + +func TestSubstrateBearer(t *testing.T) { + ctx := context.Background() + md, err := substrateBearer{token: "abc", secure: true}.GetRequestMetadata(ctx) + if err != nil || md["authorization"] != "Bearer abc" { + t.Errorf("static token: %v %v", md, err) + } + dir := t.TempDir() + f := filepath.Join(dir, "token") + if err := os.WriteFile(f, []byte(" from-file\n"), 0o600); err != nil { + t.Fatal(err) + } + md, err = substrateBearer{file: f}.GetRequestMetadata(ctx) + if err != nil || md["authorization"] != "Bearer from-file" { + t.Errorf("file token: %v %v", md, err) + } + if _, err := (substrateBearer{file: filepath.Join(dir, "missing")}).GetRequestMetadata(ctx); err == nil || !strings.Contains(err.Error(), "reading token file") { + t.Errorf("missing file: %v", err) + } + if err := os.WriteFile(f, []byte(" \n"), 0o600); err != nil { + t.Fatal(err) + } + if _, err := (substrateBearer{file: f}).GetRequestMetadata(ctx); err == nil || !strings.Contains(err.Error(), "token is empty") { + t.Errorf("empty file: %v", err) + } + if !(substrateBearer{secure: true}).RequireTransportSecurity() || (substrateBearer{}).RequireTransportSecurity() { + t.Error("RequireTransportSecurity must follow the TLS choice") + } +} + +// controlServer is the in-memory Control API the adapter is tested against. +type controlServer struct { + substrateapi.UnimplementedControlServer + fail bool + pages int +} + +func (s *controlServer) err() error { + if s.fail { + return status.Error(codes.Internal, "injected") + } + return nil +} + +func (s *controlServer) CreateActorTemplate(_ context.Context, r *substrateapi.CreateActorTemplateRequest) (*substrateapi.ActorTemplate, error) { + return r.GetActorTemplate(), s.err() +} +func (s *controlServer) DeleteActorTemplate(context.Context, *substrateapi.DeleteActorTemplateRequest) (*substrateapi.ActorTemplate, error) { + return &substrateapi.ActorTemplate{}, s.err() +} +func (s *controlServer) CreateActor(_ context.Context, r *substrateapi.CreateActorRequest) (*substrateapi.Actor, error) { + return r.GetActor(), s.err() +} +func (s *controlServer) GetActor(_ context.Context, r *substrateapi.GetActorRequest) (*substrateapi.Actor, error) { + return &substrateapi.Actor{Metadata: &substrateapi.ResourceMetadata{Name: r.GetActor().GetName()}}, s.err() +} +func (s *controlServer) ResumeActor(context.Context, *substrateapi.ResumeActorRequest) (*substrateapi.ResumeActorResponse, error) { + return &substrateapi.ResumeActorResponse{}, s.err() +} +func (s *controlServer) SuspendActor(_ context.Context, r *substrateapi.SuspendActorRequest) (*substrateapi.SuspendActorResponse, error) { + if r.GetOptions().GetMode() != substrateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS { + return nil, status.Error(codes.InvalidArgument, "mode not forwarded") + } + return &substrateapi.SuspendActorResponse{}, s.err() +} +func (s *controlServer) DeleteActor(context.Context, *substrateapi.DeleteActorRequest) (*substrateapi.Actor, error) { + return &substrateapi.Actor{}, s.err() +} +func (s *controlServer) CreateActorEgressPolicy(_ context.Context, r *substrateapi.CreateActorEgressPolicyRequest) (*substrateapi.EgressPolicy, error) { + return r.GetEgressPolicy(), s.err() +} +func (s *controlServer) ListWorkers(_ context.Context, r *substrateapi.ListWorkersRequest) (*substrateapi.ListWorkersResponse, error) { + if err := s.err(); err != nil { + return nil, err + } + s.pages++ + if r.GetOptions().GetPageToken() == "" { + return &substrateapi.ListWorkersResponse{Workers: []*substrateapi.Worker{{Metadata: &substrateapi.ResourceMetadata{Name: "w1"}}}, NextPageToken: "next"}, nil + } + return &substrateapi.ListWorkersResponse{Workers: []*substrateapi.Worker{{Metadata: &substrateapi.ResourceMetadata{Name: "w2"}}}}, nil +} + +func newBufconnControl(t *testing.T, srv substrateapi.ControlServer) *grpcControl { + t.Helper() + ln := bufconn.Listen(1 << 20) + gs := grpc.NewServer() + substrateapi.RegisterControlServer(gs, srv) + go func() { _ = gs.Serve(ln) }() + t.Cleanup(gs.Stop) + conn, err := grpc.NewClient("passthrough:///bufconn", + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return ln.DialContext(ctx) }), + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + return &grpcControl{c: substrateapi.NewControlClient(conn)} +} + +func TestGRPCControlAdapter(t *testing.T) { + srv := &controlServer{} + g := newBufconnControl(t, srv) + ctx := context.Background() + ref := &substrateapi.ObjectRef{Atespace: "hive", Name: "a1"} + if tpl, err := g.CreateActorTemplate(ctx, &substrateapi.ActorTemplate{Metadata: &substrateapi.ResourceMetadata{Name: "t"}}); err != nil || tpl.GetMetadata().GetName() != "t" { + t.Errorf("CreateActorTemplate: %v %v", tpl, err) + } + if err := g.DeleteActorTemplate(ctx, ref); err != nil { + t.Errorf("DeleteActorTemplate: %v", err) + } + if a, err := g.CreateActor(ctx, &substrateapi.Actor{Metadata: &substrateapi.ResourceMetadata{Name: "a1"}}); err != nil || a.GetMetadata().GetName() != "a1" { + t.Errorf("CreateActor: %v %v", a, err) + } + if a, err := g.GetActor(ctx, ref); err != nil || a.GetMetadata().GetName() != "a1" { + t.Errorf("GetActor: %v %v", a, err) + } + if err := g.ResumeActor(ctx, ref); err != nil { + t.Errorf("ResumeActor: %v", err) + } + if err := g.SuspendActor(ctx, ref, substrateapi.DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS); err != nil { + t.Errorf("SuspendActor: %v", err) + } + if err := g.DeleteActor(ctx, ref); err != nil { + t.Errorf("DeleteActor: %v", err) + } + if err := g.CreateActorEgressPolicy(ctx, ref, &substrateapi.EgressPolicy{}); err != nil { + t.Errorf("CreateActorEgressPolicy: %v", err) + } + workers, err := g.ListWorkers(ctx) + if err != nil || len(workers) != 2 || srv.pages != 2 { + t.Errorf("ListWorkers: %d workers, %d pages, %v", len(workers), srv.pages, err) + } + + srv.fail = true + if _, err := g.ListWorkers(ctx); status.Code(err) != codes.Internal { + t.Errorf("ListWorkers error not propagated: %v", err) + } + if _, err := g.CreateActorTemplate(ctx, nil); status.Code(err) != codes.Internal { + t.Errorf("CreateActorTemplate error not propagated: %v", err) + } +} + +// connectRouter is a TLS server that accepts CONNECT like atenet-router and +// then answers one bridge request over the tunnel. +type connectRouter struct { + t *testing.T + handler http.Handler + refuse bool + lastHdr string + lastHost string +} + +func (r *connectRouter) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if req.Method != http.MethodConnect { + http.Error(w, "not connect", http.StatusMethodNotAllowed) + return + } + r.lastHdr = req.Header.Get(substrateRouterHeader) + r.lastHost = req.Host + if r.refuse { + http.Error(w, "spiffe id not allowed", http.StatusForbidden) + return + } + hj, ok := w.(http.Hijacker) + if !ok { + r.t.Fatal("no hijacker") + } + conn, rw, err := hj.Hijack() + if err != nil { + r.t.Fatal(err) + } + defer conn.Close() + _, _ = rw.WriteString("HTTP/1.1 200 Connection established\r\n\r\n") + _ = rw.Flush() + inner, err := http.ReadRequest(bufio.NewReader(conn)) + if err != nil { + return + } + rec := httptest.NewRecorder() + r.handler.ServeHTTP(rec, inner) + resp := rec.Result() + resp.ContentLength = int64(rec.Body.Len()) + _ = resp.Write(conn) +} + +func newConnectRouter(t *testing.T, h http.Handler) (*connectRouter, *routerBridge) { + t.Helper() + cr := &connectRouter{t: t, handler: h} + srv := httptest.NewUnstartedServer(cr) + // The untrusted-certificate case below makes the server log a handshake + // error; that is the expected outcome, not noise worth printing. + srv.Config.ErrorLog = log.New(io.Discard, "", 0) + srv.StartTLS() + t.Cleanup(srv.Close) + addr := strings.TrimPrefix(srv.URL, "https://") + b := &routerBridge{ + routerAddr: addr, + tlsConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS12}, //nolint:gosec // test server + port: substratebridge.DefaultPort, + dial: (&net.Dialer{}).DialContext, + } + return cr, b +} + +func TestRouterBridgeDrivesTheBridgeProtocol(t *testing.T) { + actorWS := filepath.Join(t.TempDir(), "actor-ws") + cr, b := newConnectRouter(t, &substratebridge.Handler{Workspace: actorWS}) + ctx := context.Background() + ref := &substrateapi.ObjectRef{Atespace: "hive", Name: "hive-kick-deps-1"} + + if err := b.WaitReady(ctx, ref, time.Millisecond); err != nil { + t.Fatalf("WaitReady: %v", err) + } + if cr.lastHdr != "hive/hive-kick-deps-1" || cr.lastHost != "actor:8477" { + t.Errorf("CONNECT header=%q host=%q", cr.lastHdr, cr.lastHost) + } + + local := filepath.Join(t.TempDir(), "ws") + if err := os.MkdirAll(filepath.Join(local, "sub"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(local, "sub", "file.txt"), []byte("before"), 0o644); err != nil { + t.Fatal(err) + } + if err := b.Push(ctx, ref, local); err != nil { + t.Fatalf("Push: %v", err) + } + if got, _ := os.ReadFile(filepath.Join(actorWS, "sub", "file.txt")); string(got) != "before" { + t.Fatalf("actor workspace = %q", got) + } + var live strings.Builder + out, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"sh", "-c", "echo after > sub/file.txt; echo ran; exit 4"}}, &live) + if err != nil || out.ExitCode != 4 || !strings.Contains(out.Stdout, "ran") { + t.Fatalf("Run: %+v %v", out, err) + } + if live.String() != "ran\n" { + t.Errorf("streamed output = %q", live.String()) + } + if err := b.Pull(ctx, ref, local); err != nil { + t.Fatalf("Pull: %v", err) + } + if got, _ := os.ReadFile(filepath.Join(local, "sub", "file.txt")); strings.TrimSpace(string(got)) != "after" { + t.Errorf("local workspace after pull = %q", got) + } + if entries, _ := os.ReadDir(filepath.Dir(local)); len(entries) != 1 { + t.Errorf("staging dirs left behind: %v", entries) + } +} + +func TestRouterBridgeErrors(t *testing.T) { + ctx := context.Background() + ref := &substrateapi.ObjectRef{Atespace: "hive", Name: "a"} + + // Router refuses the CONNECT. + cr, b := newConnectRouter(t, http.NotFoundHandler()) + cr.refuse = true + if err := b.Push(ctx, ref, t.TempDir()); err == nil || !strings.Contains(err.Error(), "router refused connect") { + t.Errorf("refused connect: %v", err) + } + + // Bridge answers non-2xx. + _, b = newConnectRouter(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "unpack failed", http.StatusBadRequest) + })) + if err := b.Push(ctx, ref, t.TempDir()); err == nil || !strings.Contains(err.Error(), "400") || !strings.Contains(err.Error(), "unpack failed") { + t.Errorf("non-2xx: %v", err) + } + if _, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"x"}}, nil); err == nil { + t.Error("Run swallowed a non-2xx") + } + if err := b.Pull(ctx, ref, t.TempDir()); err == nil { + t.Error("Pull swallowed a non-2xx") + } + + // Bridge answers 200 with garbage JSON / garbage tar. + _, b = newConnectRouter(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("{not json")) + })) + if _, err := b.Run(ctx, ref, substratebridge.RunRequest{Command: []string{"x"}}, nil); err == nil || !strings.Contains(err.Error(), "decoding run response") { + t.Errorf("bad json: %v", err) + } + ws := filepath.Join(t.TempDir(), "ws") + if err := os.Mkdir(ws, 0o755); err != nil { + t.Fatal(err) + } + if err := b.Pull(ctx, ref, ws); err == nil || !strings.Contains(err.Error(), "unpacking") { + t.Errorf("bad tar: %v", err) + } + + // Push of a workspace that does not exist fails before any I/O. + if err := b.Push(ctx, ref, filepath.Join(t.TempDir(), "missing")); err == nil || !strings.Contains(err.Error(), "packing") { + t.Errorf("missing workspace: %v", err) + } + + // TLS to a server whose certificate is not trusted. + _, strict := newConnectRouter(t, http.NotFoundHandler()) + strict.tlsConfig = &tls.Config{MinVersion: tls.VersionTLS12, RootCAs: x509.NewCertPool(), ServerName: "localhost"} + if _, err := strict.tunnel(ctx, ref); err == nil || !strings.Contains(err.Error(), "tls handshake") { + t.Errorf("untrusted server: %v", err) + } + + // Dial failure. + b.dial = func(context.Context, string, string) (net.Conn, error) { return nil, errors.New("no route") } + if _, err := b.do(ctx, ref, http.MethodGet, substratebridge.PathHealth, nil); err == nil || !strings.Contains(err.Error(), "no route") { + t.Errorf("dial failure: %v", err) + } + + // Server closes the tunnel without a CONNECT response. + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer ln.Close() + _, _, serverCert := selfSignedBundle(t, t.TempDir()) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + tc := tls.Server(c, &tls.Config{Certificates: []tls.Certificate{serverCert}, MinVersion: tls.VersionTLS12}) + _ = tc.Handshake() + _ = tc.Close() + } + }() + closing := &routerBridge{routerAddr: ln.Addr().String(), tlsConfig: &tls.Config{InsecureSkipVerify: true, MinVersion: tls.VersionTLS12}, port: 1, dial: (&net.Dialer{}).DialContext} //nolint:gosec // test server + if _, err := closing.tunnel(ctx, ref); err == nil || !strings.Contains(err.Error(), "router connect") { + t.Errorf("closed tunnel: %v", err) + } +} + +func TestRouterBridgeWaitReadyGivesUpOnContext(t *testing.T) { + // The bridge answers 503 and the kick is cancelled right after that + // first probe, so WaitReady must stop without waiting for the ticker. + ctx, cancel := context.WithCancel(context.Background()) + _, b := newConnectRouter(t, http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + cancel() + http.Error(w, "starting", http.StatusServiceUnavailable) + })) + err := b.WaitReady(ctx, &substrateapi.ObjectRef{Atespace: "hive", Name: "a"}, time.Hour) + if !errors.Is(err, context.Canceled) || !strings.Contains(err.Error(), "last:") { + t.Fatalf("err = %v, want context.Canceled wrapping the last probe error", err) + } +} + +func TestRouterBridgeRequestContextCancelled(t *testing.T) { + _, b := newConnectRouter(t, http.NotFoundHandler()) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := b.do(ctx, &substrateapi.ObjectRef{}, http.MethodGet, substratebridge.PathHealth, nil); err == nil { + t.Error("do with a cancelled context succeeded") + } + if _, err := b.do(context.Background(), &substrateapi.ObjectRef{}, "bad method", "/x", nil); err == nil { + t.Error("do with an invalid method succeeded") + } +} + +func TestNewRouterBridgeLoadsCredentials(t *testing.T) { + bundle, ca, _ := selfSignedBundle(t, t.TempDir()) + b, err := newRouterBridge(SubstrateOptions{RouterClientBundle: bundle, RouterCAFile: ca, BridgePort: 9001, RouterURL: "https://router.example:8444"}) + if err != nil { + t.Fatalf("newRouterBridge: %v", err) + } + if b.routerAddr != "router.example:8444" || b.port != 9001 || len(b.tlsConfig.Certificates) != 1 || b.tlsConfig.RootCAs == nil || b.tlsConfig.ServerName != "router.example" { + t.Errorf("bridge = %+v", b) + } + // Dialer is wired to the real network. + if _, err := b.dial(context.Background(), "tcp", "127.0.0.1:1"); err == nil { + t.Error("dial to a closed port succeeded") + } +} + +func TestSubstrateReplaceDirErrors(t *testing.T) { + var good bytes.Buffer + src := t.TempDir() + if err := os.WriteFile(filepath.Join(src, "f"), []byte("x"), 0o644); err != nil { + t.Fatal(err) + } + if err := substratebridge.Pack(src, &good); err != nil { + t.Fatal(err) + } + // Parent of the workspace is not writable for the staging dir. + if err := substrateReplaceDir(bytes.NewReader(good.Bytes()), "/nonexistent-parent/ws"); err == nil { + t.Error("staging in a missing parent succeeded") + } + // Workspace itself missing: rename of the old tree fails and staging is removed. + parent := t.TempDir() + if err := substrateReplaceDir(bytes.NewReader(good.Bytes()), filepath.Join(parent, "missing")); err == nil { + t.Error("replacing a missing workspace succeeded") + } + if entries, _ := os.ReadDir(parent); len(entries) != 0 { + t.Errorf("staging left behind: %v", entries) + } +} + +// Preflight. + +type fakeSubstrateFactory struct { + fake *fakeSubstrate + err error +} + +func (f fakeSubstrateFactory) new(SubstrateOptions) (substrateClient, error) { + if f.err != nil { + return nil, f.err + } + return f.fake, nil +} + +func worker(name, state string, labels map[string]string, classes ...string) *substrateapi.Worker { + w := &substrateapi.Worker{Metadata: &substrateapi.ResourceMetadata{Name: name}, Labels: labels, Status: &substrateapi.WorkerStatus{}} + switch state { + case "active": + w.Status.State = substrateapi.WorkerState_WORKER_STATE_ACTIVE + default: + w.Status.State = substrateapi.WorkerState_WORKER_STATE_UNSPECIFIED + } + for _, c := range classes { + w.Status.SandboxClasses = append(w.Status.SandboxClasses, &substrateapi.WorkerSandboxClass{SandboxClass: c}) + } + return w +} + +func TestSubstratePreflighterWarnings(t *testing.T) { + ctx := context.Background() + opts := SubstrateOptions{Endpoint: "api:443", Token: "t"} + + p := &SubstratePreflighter{newClient: fakeSubstrateFactory{err: errors.New("no bundle")}.new} + if got := p.Check(ctx, opts); len(got) != 1 || !strings.Contains(got[0], "unreachable") || !strings.Contains(got[0], "no bundle") { + t.Errorf("constructor failure: %q", got) + } + + fake := &fakeSubstrate{listErr: errors.New("connection refused")} + p = &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + if got := p.Check(ctx, opts); len(got) != 1 || !strings.Contains(got[0], "unreachable") { + t.Errorf("list failure: %q", got) + } + if !fake.closed { + t.Error("client not closed after probe") + } + + fake = &fakeSubstrate{} + p = &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + if got := p.Check(ctx, opts); len(got) != 1 || !strings.Contains(got[0], "no active workers") { + t.Errorf("no workers: %q", got) + } + + fake = &fakeSubstrate{workers: []*substrateapi.Worker{worker("w", "active", map[string]string{"class": "microvm"}, "microvm")}} + p = &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + sel := opts + sel.WorkerSelector = map[string]string{"class": "gvisor"} + if got := p.Check(ctx, sel); len(got) != 1 || !strings.Contains(got[0], "worker_selector") { + t.Errorf("selector mismatch: %q", got) + } + + fake = &fakeSubstrate{workers: []*substrateapi.Worker{worker("w", "active", nil)}} + p = &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + if got := p.Check(ctx, opts); len(got) != 1 || !strings.Contains(got[0], "none reports a sandbox class") { + t.Errorf("no class: %q", got) + } + + fake = &fakeSubstrate{workers: []*substrateapi.Worker{ + worker("down", "unspecified", nil, "gvisor"), + worker("up", "active", map[string]string{"class": "gvisor"}, "GVISOR", ""), + }} + p = &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + if got := p.Check(ctx, opts); len(got) != 0 { + t.Errorf("healthy control plane warned: %q", got) + } +} + +func TestSubstratePreflighterCachesPerControlPlane(t *testing.T) { + ctx := context.Background() + fake := &fakeSubstrate{listErr: errors.New("down")} + now := time.Unix(1_700_000_000, 0) + p := &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new, now: func() time.Time { return now }} + a := SubstrateOptions{Endpoint: "a:443", Token: "t"} + b := SubstrateOptions{Endpoint: "b:443", Token: "t"} + p.Check(ctx, a) + p.Check(ctx, a) + p.Check(ctx, b) + if n := len(fake.calls); n != 2 { + t.Errorf("ListWorkers calls = %d, want 2 (one per control plane, second a cache hit)", n) + } + now = now.Add(substratePreflightTTL + time.Second) + p.Check(ctx, a) + if n := len(fake.calls); n != 3 { + t.Errorf("ListWorkers calls after TTL = %d, want 3", n) + } + // Returned slices are copies: mutating one must not poison the cache. + got := p.Check(ctx, a) + got[0] = "mutated" + if again := p.Check(ctx, a); again[0] == "mutated" { + t.Error("cache returned its own backing slice") + } +} + +func TestSubstrateActiveWorkersAndLabels(t *testing.T) { + n, classes := substrateActiveWorkers([]*substrateapi.Worker{ + worker("a", "active", map[string]string{"k": "v"}, "gvisor"), + worker("b", "active", map[string]string{"k": "v"}, "microvm", "gvisor"), + worker("c", "active", map[string]string{"k": "other"}, "firecracker"), + nil, + }, map[string]string{"k": "v"}) + if n != 2 || strings.Join(classes, ",") != "gvisor,microvm" { + t.Errorf("active=%d classes=%v", n, classes) + } + if !substrateLabelsMatch(nil, nil) || substrateLabelsMatch(nil, map[string]string{"a": "b"}) { + t.Error("label matching") + } +} + +func TestDefaultSubstratePreflighterUsesRealClient(t *testing.T) { + p := &SubstratePreflighter{} + got := p.Check(context.Background(), SubstrateOptions{Endpoint: "127.0.0.1:1", CAFile: "/nonexistent/ca.crt"}) + if len(got) != 1 || !strings.Contains(got[0], "unreachable") { + t.Errorf("warnings = %q", got) + } +} diff --git a/src/crux/sandbox/substrate_kicks.go b/src/crux/sandbox/substrate_kicks.go new file mode 100644 index 0000000000..cd7aba007b --- /dev/null +++ b/src/crux/sandbox/substrate_kicks.go @@ -0,0 +1,261 @@ +package sandbox + +import ( + "context" + "io" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +// Substrate kick reader (hivecommons/hive#17777), the substrate counterpart +// of the OpenShell log reader in openshell_logs.go. +// +// A substrate-runtime agent has no tmux session, so the dashboard's +// read-only kick viewer needs the agent's live actor and its output. The +// Control API has no ListActors and Substrate has no logs RPC, so neither +// can come from the control plane: the launcher, which runs in the same +// hive process as the dashboard, records each kick here instead — the +// actor's name, every state it is seen in (RESUMING, RUNNING, then +// SUSPENDED or DELETING), the worker's SPIFFE id, and the kick's stdout +// and stderr as the bridge streams /run. Unlike the OpenShell gateway log, +// this is the kick's own output and the exit code is the kick's. + +// SubstrateActorInfo is what the viewer needs to know about a kick's actor. +type SubstrateActorInfo struct { + Name string + Atespace string + // State is the actor's ActorState without its ACTOR_STATE_ prefix, or + // CREATED before the control plane has reported one. + State string + // SpiffeID is the identity of the worker the actor was scheduled on + // (WorkerAssignment.worker_spiffe_id), once known. + SpiffeID string + // Exited is true once the bridge's /run reported the kick's exit code. + Exited bool + ExitCode int + // Suspended is true when the launcher kept a snapshot of the actor + // (snapshot_on_idle) instead of deleting it. + Suspended bool + // Done is true once the launcher has finished with the actor. + Done bool +} + +// Finished reports whether the kick will produce no more output. +func (i SubstrateActorInfo) Finished() bool { return i.Done } + +// SubstrateLogReader is the dashboard's view of the substrate kicks this +// process runs. *SubstrateKicks implements it. +type SubstrateLogReader interface { + // LatestActor returns the newest kick the launcher recorded for + // launchName (see LaunchName). A finished kick is still returned so the + // viewer can say how its actor ended; found is false when the agent has + // not run a substrate kick since hive started. + LatestActor(ctx context.Context, launchName string) (info SubstrateActorInfo, found bool, err error) + // StreamRun writes the kick's output to w, calling onState for every + // state the actor reaches after the call (the caller has already shown + // the one LatestActor reported), polling every poll + // until the launcher is done with the actor or ctx ends, and returns + // the last state it observed. + StreamRun(ctx context.Context, actorName string, poll time.Duration, w io.Writer, onState func(SubstrateActorInfo)) (SubstrateActorInfo, error) +} + +// DefaultSubstrateKicks is the process-wide record the launcher writes and +// the dashboard reads. +var DefaultSubstrateKicks = &SubstrateKicks{} + +var _ SubstrateLogReader = (*SubstrateKicks)(nil) + +// SubstrateKicks records the latest substrate kick per launch name. +type SubstrateKicks struct { + mu sync.Mutex + byLaunch map[string]*substrateKick + // lastSuspended is the newest actor each launch name left suspended + // (snapshot_on_idle), with the atespace it was created in; in memory + // only, so a hive restart forgets it. + lastSuspended map[string]SubstrateActorInfo +} + +// substrateKick is one recorded kick: every state snapshot it went through +// and its output so far, capped like the bridge's. +type substrateKick struct { + history []SubstrateActorInfo + out []byte +} + +func (k *substrateKick) latest() SubstrateActorInfo { return k.history[len(k.history)-1] } + +// begin records a new kick for launchName, replacing the previous one. +func (s *SubstrateKicks) begin(launchName string, info SubstrateActorInfo) *substrateKick { + s.mu.Lock() + defer s.mu.Unlock() + if s.byLaunch == nil { + s.byLaunch = map[string]*substrateKick{} + } + k := &substrateKick{history: []SubstrateActorInfo{info}} + s.byLaunch[launchName] = k + return k +} + +// swapSuspended records actor (name and atespace) as launchName's retained +// suspended actor and returns the one it replaces, or a zero value when +// there was none. +func (s *SubstrateKicks) swapSuspended(launchName string, actor SubstrateActorInfo) SubstrateActorInfo { + s.mu.Lock() + defer s.mu.Unlock() + if s.lastSuspended == nil { + s.lastSuspended = map[string]SubstrateActorInfo{} + } + prev := s.lastSuspended[launchName] + s.lastSuspended[launchName] = SubstrateActorInfo{Name: actor.Name, Atespace: actor.Atespace} + return prev +} + +// update applies fn to the kick's latest state and records the result as +// a new snapshot when anything changed. +func (s *SubstrateKicks) update(k *substrateKick, fn func(*SubstrateActorInfo)) { + if k == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + next := k.latest() + fn(&next) + if next != k.latest() { + k.history = append(k.history, next) + } +} + +// observe records what GetActor reported: state and worker SPIFFE id. +func (s *SubstrateKicks) observe(k *substrateKick, actor *substrateapi.Actor) { + s.update(k, func(i *SubstrateActorInfo) { + if st := substrateStateName(actor.GetStatus().GetState()); st != "" { + i.State = st + } + if id := actor.GetStatus().GetWorkerAssignment().GetWorkerSpiffeId(); id != "" { + i.SpiffeID = id + } + }) +} + +// output returns a writer that appends to the kick's recorded output. +func (s *SubstrateKicks) output(k *substrateKick) io.Writer { + if k == nil { + return nil + } + return substrateKickWriter{s: s, k: k} +} + +type substrateKickWriter struct { + s *SubstrateKicks + k *substrateKick +} + +func (w substrateKickWriter) Write(p []byte) (int, error) { + w.s.mu.Lock() + defer w.s.mu.Unlock() + chunk := p + if room := substratebridge.MaxOutput - len(w.k.out); len(chunk) > room { + chunk = chunk[:max(room, 0)] + } + w.k.out = append(w.k.out, chunk...) + return len(p), nil +} + +// LatestActor implements SubstrateLogReader. +func (s *SubstrateKicks) LatestActor(_ context.Context, launchName string) (SubstrateActorInfo, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + k, ok := s.byLaunch[launchName] + if !ok { + return SubstrateActorInfo{}, false, nil + } + return k.latest(), true, nil +} + +func (s *SubstrateKicks) byActor(name string) *substrateKick { + s.mu.Lock() + defer s.mu.Unlock() + for _, k := range s.byLaunch { + if k.history[0].Name == name { + return k + } + } + return nil +} + +// StreamRun implements SubstrateLogReader. +func (s *SubstrateKicks) StreamRun(ctx context.Context, actorName string, poll time.Duration, w io.Writer, onState func(SubstrateActorInfo)) (SubstrateActorInfo, error) { + k := s.byActor(actorName) + if k == nil { + // Replaced by a newer kick before the viewer got here: this one is over. + return SubstrateActorInfo{Name: actorName, Done: true}, nil + } + ticker := time.NewTicker(poll) + defer ticker.Stop() + // The caller has just rendered LatestActor's snapshot, the newest one: + // replaying older history would show the actor going backwards. + s.mu.Lock() + seenStates, seenOut := len(k.history), 0 + s.mu.Unlock() + for { + s.mu.Lock() + states := append([]SubstrateActorInfo(nil), k.history[seenStates:]...) + out := append([]byte(nil), k.out[seenOut:]...) + seenStates, seenOut = len(k.history), len(k.out) + latest := k.latest() + s.mu.Unlock() + // States reached before the kick exited come before this poll's + // output, the rest after it, so the pane reads in order. + split := len(states) + for i, st := range states { + if st.Exited || st.Done { + split = i + break + } + } + emit := func(sts []SubstrateActorInfo) { + if onState != nil { + for _, st := range sts { + onState(st) + } + } + } + emit(states[:split]) + if len(out) > 0 { + if _, err := w.Write(out); err != nil { + return latest, err + } + } + emit(states[split:]) + if latest.Done { + return latest, nil + } + select { + case <-ctx.Done(): + return latest, ctx.Err() + case <-ticker.C: + } + } +} + +// substrateStateName is an ActorState as the viewer shows it: RUNNING, not +// ACTOR_STATE_RUNNING; empty for UNSPECIFIED. +func substrateStateName(st substrateapi.ActorState) string { + if st == substrateapi.ActorState_ACTOR_STATE_UNSPECIFIED { + return "" + } + return strings.TrimPrefix(st.String(), "ACTOR_STATE_") +} + +// Substrate actor states as SubstrateActorInfo.State carries them. +const ( + SubstrateStateCreated = "CREATED" + SubstrateStateResuming = "RESUMING" + SubstrateStateRunning = "RUNNING" + SubstrateStateSuspended = "SUSPENDED" + SubstrateStateDeleting = "DELETING" +) diff --git a/src/crux/sandbox/substrate_kicks_test.go b/src/crux/sandbox/substrate_kicks_test.go new file mode 100644 index 0000000000..50857ad705 --- /dev/null +++ b/src/crux/sandbox/substrate_kicks_test.go @@ -0,0 +1,331 @@ +package sandbox + +import ( + "context" + "encoding/json" + "errors" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +func substrateStates(t *testing.T, kicks *SubstrateKicks, launchName string) ([]SubstrateActorInfo, string) { + t.Helper() + kicks.mu.Lock() + defer kicks.mu.Unlock() + k, ok := kicks.byLaunch[launchName] + if !ok { + t.Fatalf("no kick recorded for %q", launchName) + } + return append([]SubstrateActorInfo(nil), k.history...), string(k.out) +} + +func TestSubstrateLauncherRecordsKickForTerminal(t *testing.T) { + fake := &fakeSubstrate{ + spiffeID: "spiffe://ate.local/ns/ate-workers/sa/worker", + runResp: substratebridge.RunResponse{Stdout: "step 1\n", Stderr: "warn\n", ExitCode: 5}, + } + l := newTestSubstrate(fake) + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + history, out := substrateStates(t, l.kicks, "deps-sandbox") + if out != "step 1\nwarn\n" { + t.Errorf("recorded output = %q", out) + } + var states []string + for _, h := range history { + states = append(states, h.State) + } + if got := strings.Join(states, ","); got != "CREATED,RESUMING,RUNNING,RUNNING,DELETING" { + t.Errorf("states = %s", got) + } + final := history[len(history)-1] + if final.Name != "hive-kick-deps-1700000000" || final.Atespace != "hive" || final.SpiffeID != fake.spiffeID || + !final.Exited || final.ExitCode != 5 || !final.Done || final.Suspended { + t.Errorf("final = %+v", final) + } + if info, found, err := l.kicks.LatestActor(context.Background(), "deps-sandbox"); err != nil || !found || info != final { + t.Errorf("LatestActor = %+v %v %v", info, found, err) + } + if _, found, _ := l.kicks.LatestActor(context.Background(), "other-sandbox"); found { + t.Error("an agent without a kick must not be found") + } +} + +func TestSubstrateLauncherRecordsSuspendAndFailure(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + history, _ := substrateStates(t, l.kicks, "deps-sandbox") + if final := history[len(history)-1]; final.State != SubstrateStateSuspended || !final.Suspended || !final.Done { + t.Errorf("snapshot_on_idle final = %+v", final) + } + + fake = &fakeSubstrate{waitReadyErr: errors.New("no bridge")} + l = newTestSubstrate(fake) + if _, err := l.Run(context.Background(), substrateTestSpec()); err == nil { + t.Fatal("expected bridge error") + } + history, _ = substrateStates(t, l.kicks, "deps-sandbox") + if final := history[len(history)-1]; final.State != SubstrateStateDeleting || !final.Done || final.Exited { + t.Errorf("failed kick final = %+v", final) + } + + // A kick that fails before the actor exists records nothing. + fake = &fakeSubstrate{actorErr: errors.New("quota")} + l = newTestSubstrate(fake) + _, _ = l.Run(context.Background(), substrateTestSpec()) + if _, found, _ := l.kicks.LatestActor(context.Background(), "deps-sandbox"); found { + t.Error("no actor was created; nothing must be recorded") + } +} + +// substrateStreamWriter records what StreamRun writes and closes started on +// the first write, so a test knows the stream has begun polling before it +// records further transitions. +type substrateStreamWriter struct { + strings.Builder + started chan struct{} + once sync.Once +} + +func newSubstrateStreamWriter() *substrateStreamWriter { + return &substrateStreamWriter{started: make(chan struct{})} +} + +func (w *substrateStreamWriter) Write(p []byte) (int, error) { + defer w.once.Do(func() { close(w.started) }) + return w.Builder.Write(p) +} + +func substrateStateNames(infos []SubstrateActorInfo) string { + var states []string + for _, i := range infos { + states = append(states, i.State) + } + return strings.Join(states, ",") +} + +func TestSubstrateKicksStreamRunFollowsStatesAndOutput(t *testing.T) { + kicks := &SubstrateKicks{} + k := kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a1", State: SubstrateStateCreated}) + kicks.update(k, func(i *SubstrateActorInfo) { i.State = SubstrateStateResuming }) + w := kicks.output(k) + _, _ = w.Write([]byte("line 0\n")) + + type event struct{ state, out string } + var events []event + buf := newSubstrateStreamWriter() + done := make(chan SubstrateActorInfo) + go func() { + final, err := kicks.StreamRun(context.Background(), "a1", time.Millisecond, buf, func(i SubstrateActorInfo) { + events = append(events, event{state: i.State, out: buf.String()}) + }) + if err != nil { + t.Errorf("StreamRun: %v", err) + } + done <- final + }() + <-buf.started + kicks.observe(k, &substrateapi.Actor{Status: &substrateapi.ActorStatus{ + State: substrateapi.ActorState_ACTOR_STATE_RUNNING, + WorkerAssignment: &substrateapi.WorkerAssignment{WorkerSpiffeId: "spiffe://x/w"}, + }}) + _, _ = w.Write([]byte("line 1\n")) + _, _ = w.Write([]byte("line 2\n")) + kicks.update(k, func(i *SubstrateActorInfo) { i.Exited, i.ExitCode = true, 0 }) + kicks.update(k, func(i *SubstrateActorInfo) { i.State, i.Suspended, i.Done = SubstrateStateSuspended, true, true }) + + final := <-done + if buf.String() != "line 0\nline 1\nline 2\n" { + t.Fatalf("streamed = %q", buf.String()) + } + if final.State != SubstrateStateSuspended || !final.Done || final.SpiffeID != "spiffe://x/w" { + t.Fatalf("final = %+v", final) + } + // RESUMING is what LatestActor reported before the stream started, so + // only the transitions recorded afterwards are reported. + var states []string + for _, e := range events { + states = append(states, e.state) + } + if got := strings.Join(states, ","); got != "RUNNING,RUNNING,SUSPENDED" { + t.Fatalf("states = %s", got) + } + if last := events[len(events)-1]; last.out != "line 0\nline 1\nline 2\n" { + t.Fatalf("SUSPENDED must be reported after the output, saw output %q", last.out) + } +} + +func TestSubstrateKicksStreamRunMidKickDoesNotReplayOlderStates(t *testing.T) { + kicks := &SubstrateKicks{} + k := kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a1", State: SubstrateStateCreated}) + kicks.update(k, func(i *SubstrateActorInfo) { i.State = SubstrateStateResuming }) + kicks.update(k, func(i *SubstrateActorInfo) { i.State, i.SpiffeID = SubstrateStateRunning, "spiffe://x/w" }) + w := kicks.output(k) + _, _ = w.Write([]byte("booting\n")) + + // The viewer connects while the kick is already RUNNING. + info, found, err := kicks.LatestActor(context.Background(), "deps-sandbox") + if err != nil || !found || info.State != SubstrateStateRunning { + t.Fatalf("LatestActor = %+v %v %v", info, found, err) + } + var seen []SubstrateActorInfo + buf := newSubstrateStreamWriter() + done := make(chan error) + go func() { + _, err := kicks.StreamRun(context.Background(), info.Name, time.Millisecond, buf, func(i SubstrateActorInfo) { + seen = append(seen, i) + }) + done <- err + }() + <-buf.started + kicks.update(k, func(i *SubstrateActorInfo) { i.Exited, i.ExitCode = true, 3 }) + kicks.update(k, func(i *SubstrateActorInfo) { i.State, i.Done = SubstrateStateDeleting, true }) + if err := <-done; err != nil { + t.Fatalf("StreamRun: %v", err) + } + for _, st := range seen { + if st.State == SubstrateStateCreated || st.State == SubstrateStateResuming { + t.Fatalf("a state older than RUNNING was replayed: %s", substrateStateNames(seen)) + } + } + if got := substrateStateNames(seen); got != "RUNNING,DELETING" { + t.Fatalf("states = %s", got) + } + if !seen[0].Exited || seen[0].ExitCode != 3 || !seen[1].Done { + t.Fatalf("transitions = %+v", seen) + } + if buf.String() != "booting\n" { + t.Fatalf("streamed = %q", buf.String()) + } +} + +func TestSubstrateKicksStreamRunWriteError(t *testing.T) { + kicks := &SubstrateKicks{} + k := kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a1", State: SubstrateStateRunning}) + _, _ = kicks.output(k).Write([]byte("out")) + boom := errors.New("viewer gone") + final, err := kicks.StreamRun(context.Background(), "a1", time.Millisecond, failingWriter{err: boom}, nil) + if !errors.Is(err, boom) || final.State != SubstrateStateRunning || final.Done { + t.Fatalf("StreamRun = %+v %v", final, err) + } +} + +func TestSubstrateKicksStreamRunReplacedKick(t *testing.T) { + kicks := &SubstrateKicks{} + kicks.begin("other-sandbox", SubstrateActorInfo{Name: "b1", State: SubstrateStateRunning}) + kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a1", State: SubstrateStateRunning}) + kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a2", State: SubstrateStateCreated}) + final, err := kicks.StreamRun(context.Background(), "a1", time.Millisecond, &strings.Builder{}, func(SubstrateActorInfo) { + t.Fatal("a replaced kick has no transitions to report") + }) + if err != nil || !final.Done || final.Name != "a1" { + t.Fatalf("replaced kick: %+v %v", final, err) + } +} + +func TestSubstrateLauncherDefaultsToProcessKicks(t *testing.T) { + if got := (&SubstrateLauncher{}).kickRecord(); got != DefaultSubstrateKicks { + t.Fatalf("kickRecord = %p, want DefaultSubstrateKicks", got) + } + own := &SubstrateKicks{} + if got := (&SubstrateLauncher{kicks: own}).kickRecord(); got != own { + t.Fatalf("kickRecord = %p, want %p", got, own) + } +} + +func TestSubstrateStateName(t *testing.T) { + if got := substrateStateName(substrateapi.ActorState_ACTOR_STATE_UNSPECIFIED); got != "" { + t.Errorf("UNSPECIFIED = %q", got) + } + if got := substrateStateName(substrateapi.ActorState_ACTOR_STATE_RUNNING); got != SubstrateStateRunning { + t.Errorf("RUNNING = %q", got) + } +} + +func TestSubstrateKicksStreamRunUnknownActorAndCancel(t *testing.T) { + kicks := &SubstrateKicks{} + final, err := kicks.StreamRun(context.Background(), "gone", time.Millisecond, &strings.Builder{}, nil) + if err != nil || !final.Done || final.Name != "gone" { + t.Fatalf("unknown actor: %+v %v", final, err) + } + kicks.begin("deps-sandbox", SubstrateActorInfo{Name: "a1", State: SubstrateStateRunning}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := kicks.StreamRun(ctx, "a1", time.Millisecond, &strings.Builder{}, nil); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled stream: %v", err) + } +} + +func TestSubstrateKickOutputIsCapped(t *testing.T) { + kicks := &SubstrateKicks{} + k := kicks.begin("x", SubstrateActorInfo{Name: "a"}) + w := kicks.output(k) + big := make([]byte, substratebridge.MaxOutput+10) + if n, err := w.Write(big); err != nil || n != len(big) { + t.Fatalf("Write = %d %v", n, err) + } + if n, _ := w.Write([]byte("more")); n != 4 { + t.Fatal("a full buffer must still accept writes") + } + if len(k.out) != substratebridge.MaxOutput { + t.Fatalf("recorded %d bytes", len(k.out)) + } + if kicks.output(nil) != nil { + t.Fatal("no kick, no writer") + } + kicks.update(nil, func(*SubstrateActorInfo) { t.Fatal("must not run") }) +} + +func TestSubstrateDecodeRunStreamedAndLegacy(t *testing.T) { + var live strings.Builder + stream := `{"stdout":"a","stderr":"","exit_code":0,"partial":true} +{"stdout":"","stderr":"b","exit_code":0,"partial":true} +{"stdout":"","stderr":"","exit_code":2,"error":"boom"} +` + out, err := substrateDecodeRun(strings.NewReader(stream), &live) + if err != nil || out.Stdout != "a" || out.Stderr != "b" || out.ExitCode != 2 || out.Error != "boom" || live.String() != "ab" { + t.Fatalf("streamed: %+v %v live=%q", out, err, live.String()) + } + // A bridge that predates streaming answers with one complete response. + live.Reset() + out, err = substrateDecodeRun(strings.NewReader(`{"stdout":"all","stderr":"err","exit_code":1}`), &live) + if err != nil || out.Stdout != "all" || out.Stderr != "err" || out.ExitCode != 1 || live.String() != "allerr" { + t.Fatalf("legacy: %+v %v live=%q", out, err, live.String()) + } + // A stream cut before the final frame is an error, not exit code 0. + if _, err := substrateDecodeRun(strings.NewReader(`{"stdout":"a","partial":true}`), nil); err == nil || !strings.Contains(err.Error(), "decoding run response") { + t.Fatalf("truncated: %v", err) + } +} + +func TestSubstrateDecodeRunCapsOutput(t *testing.T) { + big := strings.Repeat("o", substratebridge.MaxOutput) + var stream strings.Builder + enc := json.NewEncoder(&stream) + for _, frame := range []substratebridge.RunResponse{ + {Stdout: big, Partial: true}, + {Stdout: "more", Stderr: "e", Partial: true}, + {ExitCode: 0}, + } { + if err := enc.Encode(frame); err != nil { + t.Fatal(err) + } + } + out, err := substrateDecodeRun(strings.NewReader(stream.String()), nil) + if err != nil { + t.Fatalf("decode: %v", err) + } + if len(out.Stdout) != substratebridge.MaxOutput || strings.Contains(out.Stdout, "more") || out.Stderr != "e" { + t.Fatalf("stdout %d bytes, stderr %q", len(out.Stdout), out.Stderr) + } +} diff --git a/src/crux/sandbox/substrate_preflight.go b/src/crux/sandbox/substrate_preflight.go new file mode 100644 index 0000000000..e166591dfa --- /dev/null +++ b/src/crux/sandbox/substrate_preflight.go @@ -0,0 +1,140 @@ +package sandbox + +import ( + "context" + "fmt" + "net" + "sort" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" +) + +// Substrate cluster-prerequisite preflight (ADR-0026 phase 2; the sibling +// of the OpenShell preflight in openshell_preflight.go). The Control API +// answers whether it is up and which workers — and so which sandbox classes +// — exist; that is what decides whether a kick can land. The cluster and +// router halves in substrate_preflight_cluster.go (#17698) add the +// Kubernetes facts behind that answer when hive runs in the cluster. + +const substratePreflightTTL = 5 * time.Minute + +type substratePreflightEntry struct { + at time.Time + warnings []string +} + +// SubstratePreflighter checks a control plane and remembers the answer for a +// TTL, so the dashboard polling the same warnings does not hit it per call. +type SubstratePreflighter struct { + mu sync.Mutex + cache map[string]substratePreflightEntry + + newClient func(SubstrateOptions) (substrateClient, error) + now func() time.Time + // kube and routerDial let tests point the cluster and router halves at + // fakes; nil means the in-cluster API and a plain TCP dial. + kube *substrateKube + routerDial func(ctx context.Context, network, addr string) (net.Conn, error) +} + +// DefaultSubstratePreflighter backs the warnings AgentSandboxGateWarnings emits. +var DefaultSubstratePreflighter = &SubstratePreflighter{} + +// Check returns the warnings for one control plane. Never an error: an +// unreachable control plane is itself a warning. +func (p *SubstratePreflighter) Check(ctx context.Context, opts SubstrateOptions) []string { + now := time.Now + if p.now != nil { + now = p.now + } + key := opts.endpoint() + "\x00" + opts.atespace() + "\x00" + opts.Token + p.mu.Lock() + defer p.mu.Unlock() + if e, ok := p.cache[key]; ok && now().Sub(e.at) < substratePreflightTTL { + return append([]string(nil), e.warnings...) + } + warnings := p.probe(ctx, opts) + if p.cache == nil { + p.cache = map[string]substratePreflightEntry{} + } + p.cache[key] = substratePreflightEntry{at: now(), warnings: warnings} + return append([]string(nil), warnings...) +} + +// probe runs the three halves — Control API, cluster facts, router — and +// concatenates their warnings; a down control plane does not hide the +// cluster facts that explain it. +func (p *SubstratePreflighter) probe(ctx context.Context, opts SubstrateOptions) []string { + out := p.probeControl(ctx, opts) + out = append(out, p.probeCluster(ctx, opts)...) + return append(out, p.probeRouter(ctx, opts)...) +} + +func (p *SubstratePreflighter) probeControl(ctx context.Context, opts SubstrateOptions) []string { + ep := opts.endpoint() + ctx, cancel := context.WithTimeout(ctx, substrateRequestWait) + defer cancel() + + newClient := newSubstrateGRPCClient + if p.newClient != nil { + newClient = p.newClient + } + client, err := newClient(opts) + if err != nil { + return []string{fmt.Sprintf("substrate control api %s is unreachable (%v); every kick on the substrate runtime will fail", ep, err)} + } + defer func() { _ = client.Close() }() + + workers, err := client.Control().ListWorkers(ctx) + if err != nil { + return []string{fmt.Sprintf("substrate control api %s is unreachable (%v); every kick on the substrate runtime will fail", ep, err)} + } + active, classes := substrateActiveWorkers(workers, opts.WorkerSelector) + if active == 0 { + if len(opts.WorkerSelector) > 0 { + return []string{fmt.Sprintf("substrate control api %s has no active worker matching worker_selector %v; actors will never be scheduled — check the WorkerPool labels and that its pods are running", ep, opts.WorkerSelector)} + } + return []string{fmt.Sprintf("substrate control api %s reports no active workers; actors will never be scheduled — install a WorkerPool (see deploy/substrate.md) and check its pods are running", ep)} + } + var out []string + if len(classes) == 0 { + out = append(out, fmt.Sprintf("substrate control api %s has %d active worker(s) but none reports a sandbox class; the WorkerPool's SandboxConfig may not have resolved (kubectl get workerpool -A)", ep, active)) + } + return out +} + +// substrateActiveWorkers counts the ACTIVE workers matching the selector and +// returns the sorted set of sandbox classes they advertise. +func substrateActiveWorkers(workers []*substrateapi.Worker, selector map[string]string) (int, []string) { + active := 0 + classSet := map[string]bool{} + for _, w := range workers { + if w.GetStatus().GetState() != substrateapi.WorkerState_WORKER_STATE_ACTIVE || !substrateLabelsMatch(w.GetLabels(), selector) { + continue + } + active++ + for _, c := range w.GetStatus().GetSandboxClasses() { + if c.GetSandboxClass() != "" { + classSet[strings.ToLower(c.GetSandboxClass())] = true + } + } + } + classes := make([]string, 0, len(classSet)) + for c := range classSet { + classes = append(classes, c) + } + sort.Strings(classes) + return active, classes +} + +func substrateLabelsMatch(labels, selector map[string]string) bool { + for k, v := range selector { + if labels[k] != v { + return false + } + } + return true +} diff --git a/src/crux/sandbox/substrate_preflight_cluster.go b/src/crux/sandbox/substrate_preflight_cluster.go new file mode 100644 index 0000000000..bf9de8e81a --- /dev/null +++ b/src/crux/sandbox/substrate_preflight_cluster.go @@ -0,0 +1,534 @@ +package sandbox + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "sort" + "strconv" + "strings" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" +) + +// Cluster-level half of the Substrate preflight (hivecommons/hive#17698). +// The Control API half in substrate_preflight.go answers "is the control +// plane up and does it have workers"; this half reads the Kubernetes facts +// that explain *why* it is not, or why a kick would fail even though it is: +// the operator's CRDs and Deployments, a ready WorkerPool for the class the +// operator selected, gVisor (a SandboxConfig of class gvisor, or a runsc +// RuntimeClass) or KVM for microVMs, the certificates.k8s.io APIs behind the +// PodCertificateRequest and ClusterTrustBundle feature gates Substrate's +// SPIFFE identities need, the node kernels, and whether atenet-router +// accepts hive's client certificate. Everything is read-only, bounded by one +// request timeout per call, and skipped entirely when hive is not running in +// a cluster (no projected ServiceAccount token), where the Control API probe +// is all there is to check. + +const ( + // SubstrateDefaultOperatorNamespace is where ate-setup installs the + // control plane when the operator does not rename it. + SubstrateDefaultOperatorNamespace = "ate-system" + + substrateKubeAPIServer = "https://kubernetes.default.svc" + substrateKubeCAPath = "/var/run/secrets/kubernetes.io/serviceaccount/ca.crt" + substrateKubeMaxBody = 8 << 20 + substrateKubeNodeLimit = 500 + + substrateClassGVisor = "gvisor" + substrateClassMicroVM = "microvm" + substrateGVisorHandler = "runsc" + // substrateKVMResource is the extended resource atelet's device plugin + // advertises on nodes with /dev/kvm; the microVM class schedules on it. + substrateKVMResource = "ate.dev/kvm" + // substrateGVisorMinKernel is gVisor's documented host kernel floor. + substrateGVisorMinKernel = "4.14.77" + + substrateAteletDaemonSet = "atelet" + substrateCertsGroupPath = "/apis/certificates.k8s.io/" + substrateResourcePCR = "podcertificaterequests" + substrateResourceCTB = "clustertrustbundles" +) + +// substrateCRDs are the cluster-scoped CRDs ate-setup installs; Actors and +// ActorTemplates are Control API resources, not CRDs, so they are not here. +var substrateCRDs = []string{"workerpools.ate.dev", "sandboxconfigs.ate.dev"} + +// substrateOperatorDeployments must each have a ready replica for a kick to +// create an actor, schedule it, reach its bridge, and egress to GitHub. +var substrateOperatorDeployments = []string{"ate-api-server", "ate-controller", "atenet-router", "atenet-egress"} + +// substrateCertsVersions are the certificates.k8s.io versions that may serve +// PodCertificateRequest and ClusterTrustBundle (beta in 1.34, GA later). +var substrateCertsVersions = []string{"v1", "v1beta1"} + +// substrateKubeTokenPath is the projected Kubernetes ServiceAccount token +// the kube client authenticates with. A variable so tests pin the +// environment instead of inheriting the runner's (CI itself runs in Pods). +var substrateKubeTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token" + +// substrateInCluster reports whether hive runs in a Pod wired for Substrate: +// both the Kubernetes ServiceAccount token and hive's projected Substrate +// token exist. The kube and router halves of the preflight only run then; +// a laptop, or a CI Pod without the Substrate projection, gets the +// Control-API probe alone. +func substrateInCluster(opts SubstrateOptions) bool { + if _, err := os.Stat(substrateKubeTokenPath); err != nil { + return false + } + _, err := os.Stat(opts.tokenFile()) + return err == nil +} + +// substrateKube is the minimal read-only in-cluster API client the prover +// needs; kubejob has its own because importing it here would be a cycle. +type substrateKube struct { + server string + tokenPath string + caPath string + client *http.Client +} + +// errSubstrateKubeForbidden marks a 403: the probe stops after one warning +// rather than repeating the same RBAC gap per resource. +var errSubstrateKubeForbidden = errors.New("forbidden") + +// errSubstrateKubeNotFound marks a 404. +var errSubstrateKubeNotFound = errors.New("not found") + +func (k *substrateKube) get(ctx context.Context, path string) ([]byte, error) { + client, err := k.httpClient() + if err != nil { + return nil, err + } + reqCtx, cancel := context.WithTimeout(ctx, substrateRequestWait) + defer cancel() + req, err := http.NewRequestWithContext(reqCtx, http.MethodGet, k.server+path, nil) + if err != nil { + return nil, err + } + if raw, err := os.ReadFile(k.tokenPath); err == nil { + req.Header.Set("Authorization", "Bearer "+strings.TrimSpace(string(raw))) + } + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer func() { _ = resp.Body.Close() }() + body, err := io.ReadAll(io.LimitReader(resp.Body, substrateKubeMaxBody)) + if err != nil { + return nil, err + } + switch { + case resp.StatusCode == http.StatusForbidden: + return nil, errSubstrateKubeForbidden + case resp.StatusCode == http.StatusNotFound: + return nil, errSubstrateKubeNotFound + case resp.StatusCode/100 != 2: + return nil, fmt.Errorf("GET %s: HTTP %d", path, resp.StatusCode) + } + return body, nil +} + +func (k *substrateKube) httpClient() (*http.Client, error) { + if k.client != nil { + return k.client, nil + } + pem, err := os.ReadFile(k.caPath) + if err != nil { + return nil, fmt.Errorf("reading in-cluster CA: %w", err) + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(pem) { + return nil, errors.New("in-cluster CA bundle contained no certificates") + } + k.client = &http.Client{Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}}} + return k.client, nil +} + +// substrateOperatorNamespace derives the operator namespace from the Control +// API endpoint (`api..svc[:port]`, with or without a scheme); any other +// shape means the default install. +func substrateOperatorNamespace(endpoint string) string { + addr, _, err := substrateAddress(endpoint) + if err != nil { + return SubstrateDefaultOperatorNamespace + } + host, _, err := net.SplitHostPort(addr) + if err != nil { + host = addr + } + parts := strings.Split(host, ".") + if len(parts) >= 3 && parts[0] == "api" && parts[2] == "svc" && parts[1] != "" { + return parts[1] + } + return SubstrateDefaultOperatorNamespace +} + +// Minimal projections of the API objects the prover reads. + +type substrateKubeList struct { + Items []json.RawMessage `json:"items"` +} + +type substrateKubeWorkload struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Spec struct { + SandboxClass string `json:"sandboxClass"` + SandboxClasses []struct { + Name string `json:"name"` + } `json:"sandboxClasses"` + } `json:"spec"` + Status struct { + ReadyReplicas int `json:"readyReplicas"` + NumberReady int `json:"numberReady"` + } `json:"status"` + Handler string `json:"handler"` +} + +type substrateKubeNode struct { + Metadata struct { + Name string `json:"name"` + } `json:"metadata"` + Status struct { + Allocatable map[string]string `json:"allocatable"` + NodeInfo struct { + KernelVersion string `json:"kernelVersion"` + } `json:"nodeInfo"` + } `json:"status"` +} + +type substrateKubeResourceList struct { + Resources []struct { + Name string `json:"name"` + } `json:"resources"` +} + +// clusterProbe holds one probe's state so the step functions stay short. +type substrateClusterProbe struct { + kube *substrateKube + ns string + sel map[string]string + out []string + // classes are the sandbox classes the ready WorkerPools offer; gvisor and + // kvm record whether the cluster can run each class. + classes map[string]bool + gvisor bool + kvm bool +} + +// substrateKubeItems GETs a collection and decodes each item into T. +func substrateKubeItems[T any](ctx context.Context, k *substrateKube, path string) ([]T, error) { + body, err := k.get(ctx, path) + if err != nil { + return nil, err + } + var list substrateKubeList + if err := json.Unmarshal(body, &list); err != nil { + return nil, fmt.Errorf("GET %s: %w", path, err) + } + items := make([]T, 0, len(list.Items)) + for _, raw := range list.Items { + var item T + if err := json.Unmarshal(raw, &item); err != nil { + return nil, fmt.Errorf("GET %s: %w", path, err) + } + items = append(items, item) + } + return items, nil +} + +// warnf appends one warning. +func (c *substrateClusterProbe) warnf(format string, args ...any) { + c.out = append(c.out, fmt.Sprintf(format, args...)) +} + +// probeCluster runs the Kubernetes checks and returns their warnings. A 403 +// anywhere ends the probe with one warning naming the RBAC the prover needs. +func (p *SubstratePreflighter) probeCluster(ctx context.Context, opts SubstrateOptions) []string { + kube := p.kube + if kube == nil { + if !substrateInCluster(opts) { + return nil + } + kube = &substrateKube{server: substrateKubeAPIServer, tokenPath: substrateKubeTokenPath, caPath: substrateKubeCAPath} + } + c := &substrateClusterProbe{kube: kube, ns: substrateOperatorNamespace(opts.endpoint()), sel: opts.WorkerSelector, classes: map[string]bool{}} + steps := []func(context.Context) error{c.crds, c.operator, c.pools, c.sandboxClasses, c.featureGates, c.nodes} + for _, step := range steps { + if err := step(ctx); err != nil { + if errors.Is(err, errSubstrateCRDsMissing) { + return c.out + } + if errors.Is(err, errSubstrateKubeForbidden) { + return append(c.out, "hive's ServiceAccount may not read the Substrate cluster facts (403); the operator, WorkerPool, gVisor and feature-gate checks were skipped — apply deploy/k8s/sandbox-substrate-preflight-rbac.yaml") + } + c.warnf("substrate cluster facts could not be read (%v); the operator, WorkerPool, gVisor and feature-gate checks were skipped", err) + return c.out + } + } + return c.out +} + +// crds checks the operator's CRDs are installed; missing ones end the probe +// early, because every later check would just restate "not installed". +func (c *substrateClusterProbe) crds(ctx context.Context) error { + missing := 0 + for _, name := range substrateCRDs { + _, err := c.kube.get(ctx, "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/"+name) + switch { + case err == nil: + case errors.Is(err, errSubstrateKubeNotFound): + missing++ + c.warnf("substrate CRD %s is not installed; the Substrate operator is absent and every kick on the substrate runtime will fail — run `ate-setup deploy ate-system` (see deploy/substrate.md)", name) + default: + return err + } + } + if missing > 0 { + return errSubstrateCRDsMissing + } + return nil +} + +// errSubstrateCRDsMissing stops the probe after the CRD warnings. +var errSubstrateCRDsMissing = errors.New("crds missing") + +// operator checks each control-plane Deployment and the atelet DaemonSet +// have a ready replica in the operator namespace. +func (c *substrateClusterProbe) operator(ctx context.Context) error { + deps, err := substrateKubeItems[substrateKubeWorkload](ctx, c.kube, "/apis/apps/v1/namespaces/"+c.ns+"/deployments") + if err != nil { + return err + } + ready := map[string]int{} + for _, d := range deps { + ready[d.Metadata.Name] = d.Status.ReadyReplicas + } + for _, name := range substrateOperatorDeployments { + n, ok := ready[name] + switch { + case !ok: + c.warnf("substrate operator deployment %s/%s is not installed; run `ate-setup deploy ate-system` (see deploy/substrate.md)", c.ns, name) + case n == 0: + c.warnf("substrate operator deployment %s/%s has no ready replica; kicks on the substrate runtime will fail until it is up (kubectl -n %s describe deploy %s)", c.ns, name, c.ns, name) + } + } + body, err := c.kube.get(ctx, "/apis/apps/v1/namespaces/"+c.ns+"/daemonsets/"+substrateAteletDaemonSet) + switch { + case errors.Is(err, errSubstrateKubeNotFound): + c.warnf("substrate node agent daemonset %s/%s is not installed; no node can host a worker — run `ate-setup deploy ate-system`", c.ns, substrateAteletDaemonSet) + return nil + case err != nil: + return err + } + var ds substrateKubeWorkload + if err := json.Unmarshal(body, &ds); err != nil { + return fmt.Errorf("daemonset %s: %w", substrateAteletDaemonSet, err) + } + if ds.Status.NumberReady == 0 { + c.warnf("substrate node agent daemonset %s/%s has no ready pod; workers cannot start (kubectl -n %s describe ds %s)", c.ns, substrateAteletDaemonSet, c.ns, substrateAteletDaemonSet) + } + return nil +} + +// pools checks at least one WorkerPool has a ready replica and records the +// sandbox classes those pools offer. +func (c *substrateClusterProbe) pools(ctx context.Context) error { + pools, err := substrateKubeItems[substrateKubeWorkload](ctx, c.kube, "/apis/ate.dev/v1/workerpools") + if err != nil { + return err + } + ready := 0 + for _, p := range pools { + if p.Status.ReadyReplicas == 0 { + continue + } + ready++ + for _, sc := range p.Spec.SandboxClasses { + c.classes[strings.ToLower(sc.Name)] = true + } + } + if len(pools) == 0 { + c.warnf("no substrate WorkerPool exists; actors have nowhere to run — create one (see deploy/substrate.md)") + } else if ready == 0 { + c.warnf("none of the %d substrate WorkerPool(s) has a ready replica; actors will never be scheduled (kubectl get workerpools -A)", len(pools)) + } + return nil +} + +// sandboxClasses checks the cluster can actually run the classes the pools +// offer: gVisor via a SandboxConfig of class gvisor (atelet fetches runsc +// itself) or a runsc RuntimeClass; microVMs via atelet's ate.dev/kvm +// resource on at least one node (checked in nodes). +func (c *substrateClusterProbe) sandboxClasses(ctx context.Context) error { + cfgs, err := substrateKubeItems[substrateKubeWorkload](ctx, c.kube, "/apis/ate.dev/v1/sandboxconfigs") + if err != nil { + return err + } + for _, cfg := range cfgs { + if strings.EqualFold(cfg.Spec.SandboxClass, substrateClassGVisor) { + c.gvisor = true + } + } + rcs, err := substrateKubeItems[substrateKubeWorkload](ctx, c.kube, "/apis/node.k8s.io/v1/runtimeclasses") + if err != nil && !errors.Is(err, errSubstrateKubeNotFound) { + return err + } + for _, rc := range rcs { + if rc.Handler == substrateGVisorHandler { + c.gvisor = true + } + } + wantGVisor := c.classes[substrateClassGVisor] || len(c.classes) == 0 + if wantGVisor && !c.gvisor { + c.warnf("cluster has no gVisor: no SandboxConfig with sandboxClass %s and no RuntimeClass with handler %s; Substrate refuses plain containers, so a gvisor WorkerPool cannot start workers — `ate-setup deploy ate-system` installs the gvisor-default SandboxConfig", substrateClassGVisor, substrateGVisorHandler) + } + return nil +} + +// featureGates checks the certificates.k8s.io APIs that only exist when the +// PodCertificateRequest and ClusterTrustBundle feature gates are on; without +// them atelet, atenet-router and hive's own router client get no SPIFFE +// certificates and every CONNECT to an actor fails. +func (c *substrateClusterProbe) featureGates(ctx context.Context) error { + served := map[string]bool{} + for _, v := range substrateCertsVersions { + body, err := c.kube.get(ctx, substrateCertsGroupPath+v) + if errors.Is(err, errSubstrateKubeNotFound) { + continue + } + if err != nil { + return err + } + var rl substrateKubeResourceList + if err := json.Unmarshal(body, &rl); err != nil { + return fmt.Errorf("certificates.k8s.io/%s: %w", v, err) + } + for _, r := range rl.Resources { + served[strings.ToLower(r.Name)] = true + } + } + if !served[substrateResourcePCR] { + c.warnf("the PodCertificateRequest API is not served; Substrate's SPIFFE identities (atelet, atenet-router, hive's router client) cannot be issued — enable the PodCertificateRequest feature gate and certificates.k8s.io/v1beta1 on the API server and kubelets (Kubernetes ≥ 1.34)") + } + if !served[substrateResourceCTB] { + c.warnf("the ClusterTrustBundle API is not served; the Control API and router CA bundles cannot be projected into hive — enable the ClusterTrustBundle feature gate and certificates.k8s.io/v1beta1 on the API server and kubelets (Kubernetes ≥ 1.34)") + } + return nil +} + +// nodes surveys the node kernels against gVisor's floor and records whether +// any node advertises KVM for the microVM class. +func (c *substrateClusterProbe) nodes(ctx context.Context) error { + nodes, err := substrateKubeItems[substrateKubeNode](ctx, c.kube, "/api/v1/nodes?limit="+strconv.Itoa(substrateKubeNodeLimit)) + if err != nil { + return err + } + var old []string + for _, n := range nodes { + if _, ok := n.Status.Allocatable[substrateKVMResource]; ok { + c.kvm = true + } + if kv := n.Status.NodeInfo.KernelVersion; kv != "" && substrateKernelLess(kv, substrateGVisorMinKernel) { + old = append(old, n.Metadata.Name+" ("+kv+")") + } + } + if len(old) > 0 { + sort.Strings(old) + c.warnf("%d node(s) run a kernel older than gVisor's %s floor and cannot host a gvisor worker: %s", len(old), substrateGVisorMinKernel, strings.Join(old, ", ")) + } + if c.classes[substrateClassMicroVM] && !c.kvm { + c.warnf("a WorkerPool offers the %s class but no node advertises %s; microVM workers need /dev/kvm (KVM-capable shapes) and atelet's device plugin", substrateClassMicroVM, substrateKVMResource) + } + return nil +} + +// substrateKernelLess reports whether kernel release a (e.g. 5.15.0-1 or +// 6.12.0-207.el9uek.aarch64) is older than the dotted floor b. A release the +// prover cannot parse is never reported as old. +func substrateKernelLess(a, b string) bool { + pa, oka := substrateKernelParts(a) + pb, okb := substrateKernelParts(b) + if !oka || !okb { + return false + } + for i := range pa { + if pa[i] != pb[i] { + return pa[i] < pb[i] + } + } + return false +} + +func substrateKernelParts(s string) ([3]int, bool) { + var out [3]int + s, _, _ = strings.Cut(s, "-") + parts := strings.Split(s, ".") + if len(parts) < 2 { + return out, false + } + for i := 0; i < 3 && i < len(parts); i++ { + n, err := strconv.Atoi(parts[i]) + if err != nil { + return out, false + } + out[i] = n + } + return out, true +} + +// substrateRouterProbeActor is the (nonexistent) actor the preflight names +// in its CONNECT: the router answers with an HTTP status once it has +// accepted hive's client certificate, and with a TLS alert when it has not. +const substrateRouterProbeActor = "hive-preflight" + +// probeRouter checks hive can reach atenet-router and that the router +// accepts its projected client bundle: the same mTLS CONNECT the kick's +// bridge tunnel makes, aimed at an actor that does not exist. Any HTTP +// answer means the certificate was accepted (TLS 1.3 reports a rejected +// client certificate only after the handshake, so the handshake alone is +// not proof). Skipped outside a cluster, where neither the bundle nor the +// router exist. +func (p *SubstratePreflighter) probeRouter(ctx context.Context, opts SubstrateOptions) []string { + if p.kube == nil && !substrateInCluster(opts) { + return nil + } + bridge, err := newRouterBridge(opts) + if err != nil { + return []string{fmt.Sprintf("atenet-router client credentials are not usable (%v); every kick on the substrate runtime will fail at the bridge tunnel", err)} + } + if p.routerDial != nil { + bridge.dial = p.routerDial + } + ctx, cancel := context.WithTimeout(ctx, substrateRequestWait) + defer cancel() + ref := &substrateapi.ObjectRef{Atespace: opts.atespace(), Name: substrateRouterProbeActor} + conn, err := bridge.tunnel(ctx, ref) + if conn != nil { + _ = conn.Close() + } + switch { + case err == nil, strings.Contains(err.Error(), "router refused connect"): + return nil + case strings.Contains(err.Error(), "router tls handshake") || strings.Contains(err.Error(), "router connect"): + return []string{fmt.Sprintf("atenet-router %s rejected hive's client certificate (%v); check the podCertificate projection at %s and --ingress-allowed-spiffe-ids on the router", bridge.routerAddr, err, substrateRouterBundlePath(opts))} + } + return []string{fmt.Sprintf("atenet-router %s is unreachable (%v); every kick on the substrate runtime will fail at the bridge tunnel — check the atenet-router Service and NetworkPolicies", bridge.routerAddr, err)} +} + +func substrateRouterBundlePath(opts SubstrateOptions) string { + if opts.RouterClientBundle != "" { + return opts.RouterClientBundle + } + return SubstrateDefaultRouterClientBundle +} diff --git a/src/crux/sandbox/substrate_preflight_cluster_test.go b/src/crux/sandbox/substrate_preflight_cluster_test.go new file mode 100644 index 0000000000..0cefac58f0 --- /dev/null +++ b/src/crux/sandbox/substrate_preflight_cluster_test.go @@ -0,0 +1,513 @@ +package sandbox + +import ( + "bufio" + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strconv" + "strings" + "testing" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" +) + +// Cluster/router half of the Substrate preflight (#17698): a fake Kubernetes +// API serves the handful of read-only endpoints the prover walks, and a TLS +// listener plays atenet-router. + +// fakeKubeAPI is a path → JSON (or status code) table. +type fakeKubeAPI struct { + bodies map[string]any + codes map[string]int + calls []string +} + +func (f *fakeKubeAPI) handler() http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.calls = append(f.calls, r.URL.Path) + if code, ok := f.codes[r.URL.Path]; ok { + w.WriteHeader(code) + return + } + body, ok := f.bodies[r.URL.Path] + if !ok { + w.WriteHeader(http.StatusNotFound) + return + } + if raw, isRaw := body.(string); isRaw { + _, _ = w.Write([]byte(raw)) + return + } + _ = json.NewEncoder(w).Encode(body) + }) +} + +func kubeList(items ...any) map[string]any { return map[string]any{"items": items} } + +func kubeDeploy(name string, ready int) map[string]any { + return map[string]any{"metadata": map[string]any{"name": name}, "status": map[string]any{"readyReplicas": ready}} +} + +func kubePool(name string, ready int, classes ...string) map[string]any { + var sc []map[string]any + for _, c := range classes { + sc = append(sc, map[string]any{"name": c}) + } + return map[string]any{"metadata": map[string]any{"name": name}, "spec": map[string]any{"sandboxClasses": sc}, "status": map[string]any{"readyReplicas": ready}} +} + +func kubeNode(name, kernel string, kvm bool) map[string]any { + alloc := map[string]string{"cpu": "4"} + if kvm { + alloc[substrateKVMResource] = "1" + } + return map[string]any{"metadata": map[string]any{"name": name}, "status": map[string]any{"allocatable": alloc, "nodeInfo": map[string]any{"kernelVersion": kernel}}} +} + +func kubeResources(names ...string) map[string]any { + var rs []map[string]any + for _, n := range names { + rs = append(rs, map[string]any{"name": n}) + } + return map[string]any{"resources": rs} +} + +const ( + pathCRDs = "/apis/apiextensions.k8s.io/v1/customresourcedefinitions/" + pathDeploys = "/apis/apps/v1/namespaces/ate-system/deployments" + pathAtelet = "/apis/apps/v1/namespaces/ate-system/daemonsets/atelet" + pathPools = "/apis/ate.dev/v1/workerpools" + pathSandboxes = "/apis/ate.dev/v1/sandboxconfigs" + pathRuntimes = "/apis/node.k8s.io/v1/runtimeclasses" + pathNodes = "/api/v1/nodes" +) + +// healthyKube is a cluster where every check passes. +func healthyKube() *fakeKubeAPI { + f := &fakeKubeAPI{bodies: map[string]any{}, codes: map[string]int{}} + for _, crd := range substrateCRDs { + f.bodies[pathCRDs+crd] = map[string]any{"metadata": map[string]any{"name": crd}} + } + var deps []any + for _, d := range substrateOperatorDeployments { + deps = append(deps, kubeDeploy(d, 1)) + } + f.bodies[pathDeploys] = kubeList(deps...) + f.bodies[pathAtelet] = map[string]any{"status": map[string]any{"numberReady": 3}} + f.bodies[pathPools] = kubeList(kubePool("gv", 2, "gvisor")) + f.bodies[pathSandboxes] = kubeList(map[string]any{"spec": map[string]any{"sandboxClass": "gvisor"}}) + f.bodies[pathRuntimes] = kubeList() + f.bodies[substrateCertsGroupPath+"v1beta1"] = kubeResources("podcertificaterequests", "clustertrustbundles") + f.bodies[pathNodes] = kubeList(kubeNode("n1", "6.12.0-207.el9uek.aarch64", false)) + return f +} + +// clusterPreflighter returns a preflighter whose Control API half is healthy +// and whose cluster half talks to the fake, with the router half stubbed to +// a refused dial unless the test overrides it. +func clusterPreflighter(t *testing.T, f *fakeKubeAPI) (*SubstratePreflighter, SubstrateOptions) { + t.Helper() + srv := httptest.NewServer(f.handler()) + t.Cleanup(srv.Close) + dir := t.TempDir() + bundle, ca, _ := selfSignedBundle(t, dir) + token := filepath.Join(dir, "token") + if err := os.WriteFile(token, []byte("sa-token"), 0o600); err != nil { + t.Fatal(err) + } + fake := &fakeSubstrate{workers: []*substrateapi.Worker{worker("w", "active", nil, "gvisor")}} + p := &SubstratePreflighter{ + newClient: fakeSubstrateFactory{fake: fake}.new, + kube: &substrateKube{server: srv.URL, tokenPath: token, caPath: ca, client: srv.Client()}, + routerDial: func(context.Context, string, string) (net.Conn, error) { + return nil, errors.New("dial stubbed") + }, + } + opts := SubstrateOptions{Endpoint: "api.ate-system.svc:443", Token: "t", RouterClientBundle: bundle, RouterCAFile: ca} + return p, opts +} + +func TestSubstratePreflightClusterHealthyOnlyWarnsAboutRouter(t *testing.T) { + f := healthyKube() + p, opts := clusterPreflighter(t, f) + got := p.Check(context.Background(), opts) + if len(got) != 1 || !strings.Contains(got[0], "atenet-router") || !strings.Contains(got[0], "dial stubbed") { + t.Fatalf("healthy cluster warnings = %q", got) + } + if !strings.Contains(strings.Join(f.calls, " "), pathNodes) { + t.Errorf("node survey not reached: %v", f.calls) + } +} + +func TestSubstratePreflightClusterMissingCRDsStopsEarly(t *testing.T) { + f := healthyKube() + delete(f.bodies, pathCRDs+"workerpools.ate.dev") + p, opts := clusterPreflighter(t, f) + got := p.Check(context.Background(), opts) + var crd int + for _, w := range got { + if strings.Contains(w, "CRD workerpools.ate.dev is not installed") { + crd++ + } + if strings.Contains(w, "deployment") || strings.Contains(w, "WorkerPool") { + t.Errorf("operator checks ran after missing CRDs: %q", w) + } + } + if crd != 1 { + t.Errorf("CRD warnings = %d in %q", crd, got) + } +} + +func TestSubstratePreflightClusterForbiddenIsOneWarning(t *testing.T) { + f := healthyKube() + f.codes[pathDeploys] = http.StatusForbidden + p, opts := clusterPreflighter(t, f) + got := p.Check(context.Background(), opts) + n := 0 + for _, w := range got { + if strings.Contains(w, "403") && strings.Contains(w, "sandbox-substrate-preflight-rbac.yaml") { + n++ + } + } + if n != 1 || len(got) != 2 { + t.Errorf("forbidden warnings = %q", got) + } +} + +func TestSubstratePreflightClusterReadErrorIsAWarning(t *testing.T) { + f := healthyKube() + f.codes[pathPools] = http.StatusInternalServerError + p, opts := clusterPreflighter(t, f) + got := p.Check(context.Background(), opts) + if len(got) != 2 || !strings.Contains(got[0], "could not be read") || !strings.Contains(got[0], "HTTP 500") { + t.Errorf("read error warnings = %q", got) + } + + f = healthyKube() + f.bodies[pathPools] = "not json" + p, opts = clusterPreflighter(t, f) + if got := p.Check(context.Background(), opts); len(got) != 2 || !strings.Contains(got[0], "could not be read") { + t.Errorf("bad json warnings = %q", got) + } + + f = healthyKube() + f.bodies[pathPools] = kubeList("not an object") + p, opts = clusterPreflighter(t, f) + if got := p.Check(context.Background(), opts); len(got) != 2 || !strings.Contains(got[0], "could not be read") { + t.Errorf("bad item warnings = %q", got) + } + + f = healthyKube() + f.bodies[pathAtelet] = "not json" + p, opts = clusterPreflighter(t, f) + if got := p.Check(context.Background(), opts); len(got) != 2 || !strings.Contains(got[0], "daemonset atelet") { + t.Errorf("bad daemonset warnings = %q", got) + } + + f = healthyKube() + f.bodies[substrateCertsGroupPath+"v1beta1"] = "not json" + p, opts = clusterPreflighter(t, f) + if got := p.Check(context.Background(), opts); len(got) != 2 || !strings.Contains(got[0], "certificates.k8s.io/v1beta1") { + t.Errorf("bad discovery warnings = %q", got) + } +} + +func TestSubstratePreflightClusterOperatorAndPoolWarnings(t *testing.T) { + f := healthyKube() + f.bodies[pathDeploys] = kubeList(kubeDeploy("ate-api-server", 0), kubeDeploy("ate-controller", 1), kubeDeploy("atenet-router", 1)) + f.bodies[pathAtelet] = map[string]any{"status": map[string]any{"numberReady": 0}} + f.bodies[pathPools] = kubeList(kubePool("gv", 0, "gvisor")) + p, opts := clusterPreflighter(t, f) + got := strings.Join(p.Check(context.Background(), opts), "\n") + for _, want := range []string{ + "deployment ate-system/ate-api-server has no ready replica", + "deployment ate-system/atenet-egress is not installed", + "daemonset ate-system/atelet has no ready pod", + "none of the 1 substrate WorkerPool(s) has a ready replica", + } { + if !strings.Contains(got, want) { + t.Errorf("missing %q in:\n%s", want, got) + } + } + if strings.Contains(got, "ate-controller") || strings.Contains(got, "atenet-router is") { + t.Errorf("healthy deployments warned:\n%s", got) + } + + f = healthyKube() + delete(f.bodies, pathAtelet) + f.bodies[pathPools] = kubeList() + p, opts = clusterPreflighter(t, f) + got = strings.Join(p.Check(context.Background(), opts), "\n") + if !strings.Contains(got, "daemonset ate-system/atelet is not installed") || !strings.Contains(got, "no substrate WorkerPool exists") { + t.Errorf("missing atelet/pool warnings:\n%s", got) + } +} + +func TestSubstratePreflightClusterSandboxClassWarnings(t *testing.T) { + // No gvisor SandboxConfig and no runsc RuntimeClass → warning. + f := healthyKube() + f.bodies[pathSandboxes] = kubeList() + delete(f.bodies, pathRuntimes) + p, opts := clusterPreflighter(t, f) + got := strings.Join(p.Check(context.Background(), opts), "\n") + if !strings.Contains(got, "cluster has no gVisor") { + t.Errorf("missing gVisor warning:\n%s", got) + } + + // A runsc RuntimeClass alone satisfies gVisor. + f = healthyKube() + f.bodies[pathSandboxes] = kubeList() + f.bodies[pathRuntimes] = kubeList(map[string]any{"handler": "runsc"}, map[string]any{"handler": "kata"}) + p, opts = clusterPreflighter(t, f) + if got := strings.Join(p.Check(context.Background(), opts), "\n"); strings.Contains(got, "gVisor") { + t.Errorf("runsc RuntimeClass not honoured:\n%s", got) + } + + // A microvm pool with no KVM node warns; a gvisor-less pool set does not + // demand gVisor. + f = healthyKube() + f.bodies[pathPools] = kubeList(kubePool("vm", 1, "microvm")) + f.bodies[pathSandboxes] = kubeList() + p, opts = clusterPreflighter(t, f) + got = strings.Join(p.Check(context.Background(), opts), "\n") + if !strings.Contains(got, "no node advertises ate.dev/kvm") || strings.Contains(got, "no gVisor") { + t.Errorf("microvm warnings:\n%s", got) + } + f.bodies[pathNodes] = kubeList(kubeNode("n1", "6.12.0", true)) + p, opts = clusterPreflighter(t, f) + if got := strings.Join(p.Check(context.Background(), opts), "\n"); strings.Contains(got, "kvm") { + t.Errorf("KVM node not honoured:\n%s", got) + } +} + +func TestSubstratePreflightClusterFeatureGatesAndKernels(t *testing.T) { + f := healthyKube() + delete(f.bodies, substrateCertsGroupPath+"v1beta1") + f.bodies[pathNodes] = kubeList(kubeNode("old", "4.14.0-1", false), kubeNode("new", "5.15.0", false), kubeNode("odd", "weird", false)) + p, opts := clusterPreflighter(t, f) + got := strings.Join(p.Check(context.Background(), opts), "\n") + for _, want := range []string{"PodCertificateRequest API is not served", "ClusterTrustBundle API is not served", "1 node(s) run a kernel older", "old (4.14.0-1)"} { + if !strings.Contains(got, want) { + t.Errorf("missing %q in:\n%s", want, got) + } + } + + // GA v1 serving both resources is enough. + f = healthyKube() + delete(f.bodies, substrateCertsGroupPath+"v1beta1") + f.bodies[substrateCertsGroupPath+"v1"] = kubeResources("PodCertificateRequests", "ClusterTrustBundles") + p, opts = clusterPreflighter(t, f) + if got := strings.Join(p.Check(context.Background(), opts), "\n"); strings.Contains(got, "not served") { + t.Errorf("v1 discovery not honoured:\n%s", got) + } +} + +// init pins the Kubernetes token path to a file that does not exist so the +// Control-API-only tests behave the same on a laptop and in a CI Pod (where +// the runner has a real ServiceAccount token). Tests that want the +// in-cluster halves inject kube/routerDial or call inClusterToken. +func init() { + substrateKubeTokenPath = filepath.Join(os.TempDir(), "hive-substrate-preflight-no-token-"+strconv.Itoa(os.Getpid())) +} + +// inClusterToken makes the prover believe it runs in a Pod wired for +// Substrate: a Kubernetes token and a projected Substrate token both exist. +// It returns the Substrate token path to set as opts.TokenFile. +func inClusterToken(t *testing.T) string { + t.Helper() + dir := t.TempDir() + prev := substrateKubeTokenPath + substrateKubeTokenPath = filepath.Join(dir, "k8s-token") + ate := filepath.Join(dir, "ate-token") + for _, f := range []string{substrateKubeTokenPath, ate} { + if err := os.WriteFile(f, []byte("t"), 0o600); err != nil { + t.Fatal(err) + } + } + t.Cleanup(func() { substrateKubeTokenPath = prev }) + return ate +} + +func TestSubstratePreflightClusterSkippedOutsideCluster(t *testing.T) { + fake := &fakeSubstrate{workers: []*substrateapi.Worker{worker("w", "active", nil, "gvisor")}} + p := &SubstratePreflighter{newClient: fakeSubstrateFactory{fake: fake}.new} + if got := p.Check(context.Background(), SubstrateOptions{Endpoint: "api:443"}); len(got) != 0 { + t.Errorf("outside a cluster the kube/router halves must be silent: %q", got) + } + + // In a Pod with no reachable API server the kube half reports once and + // the router half reports unusable credentials, instead of staying silent. + ate := inClusterToken(t) + // Only one of the two markers present is still "outside". + if substrateInCluster(SubstrateOptions{}) { + t.Error("a Kubernetes token alone must not count as in-cluster") + } + opts := SubstrateOptions{Endpoint: "api:443", TokenFile: ate, RouterClientBundle: filepath.Join(t.TempDir(), "missing.pem")} + got := p.probeCluster(context.Background(), opts) + if len(got) != 1 || !strings.Contains(got[0], "cluster facts could not be read") { + t.Errorf("in-cluster without an API server: %q", got) + } + if got := p.probeRouter(context.Background(), opts); len(got) != 1 || !strings.Contains(got[0], "not usable") { + t.Errorf("in-cluster without a bundle: %q", got) + } +} + +func TestSubstratePreflightKubeClientErrors(t *testing.T) { + k := &substrateKube{server: "https://127.0.0.1:1", tokenPath: "/nonexistent", caPath: "/nonexistent"} + if _, err := k.get(context.Background(), "/x"); err == nil || !strings.Contains(err.Error(), "in-cluster CA") { + t.Errorf("missing CA: %v", err) + } + dir := t.TempDir() + bad := filepath.Join(dir, "ca.crt") + _ = os.WriteFile(bad, []byte("not pem"), 0o600) + k = &substrateKube{server: "https://127.0.0.1:1", tokenPath: "/nonexistent", caPath: bad} + if _, err := k.get(context.Background(), "/x"); err == nil || !strings.Contains(err.Error(), "no certificates") { + t.Errorf("bad CA: %v", err) + } + _, ca, _ := selfSignedBundle(t, dir) + k = &substrateKube{server: "https://127.0.0.1:1", tokenPath: "/nonexistent", caPath: ca} + if _, err := k.get(context.Background(), "/x"); err == nil { + t.Error("refused dial did not error") + } + if k.client == nil { + t.Error("client not cached") + } + k = &substrateKube{server: "http://[::1]:namedport", client: http.DefaultClient} + if _, err := k.get(context.Background(), "/x"); err == nil { + t.Error("bad URL did not error") + } +} + +func TestSubstrateOperatorNamespace(t *testing.T) { + for in, want := range map[string]string{ + "api.ate-system.svc:443": "ate-system", + "https://api.substrate-prod.svc.cluster.local": "substrate-prod", + "grpc://api.hive-sub.svc:8080": "hive-sub", + "": "ate-system", + "control.example.com:443": "ate-system", + "api..svc:443": "ate-system", + "not a url": "ate-system", + } { + if got := substrateOperatorNamespace(in); got != want { + t.Errorf("%q → %q, want %q", in, got, want) + } + } +} + +func TestSubstrateKernelLess(t *testing.T) { + cases := []struct { + a, b string + want bool + }{ + {"4.14.0-1", "4.14.77", true}, + {"4.14.77", "4.14.77", false}, + {"5.15.0-1.el8uek", "4.14.77", false}, + {"6.12.0-207.el9uek.aarch64", "4.14.77", false}, + {"4.9", "4.14.77", true}, + {"weird", "4.14.77", false}, + {"4.x.1", "4.14.77", false}, + {"5.15", "x", false}, + } + for _, c := range cases { + if got := substrateKernelLess(c.a, c.b); got != c.want { + t.Errorf("substrateKernelLess(%q, %q) = %v", c.a, c.b, got) + } + } +} + +// fakeRouterListener accepts TLS connections requiring a client certificate +// signed by the test CA and answers any CONNECT with 404, so the probe sees +// exactly what atenet-router would show an accepted or rejected client. +func fakeRouterListener(t *testing.T, cert tls.Certificate) net.Listener { + t.Helper() + pool := x509.NewCertPool() + leaf, _ := x509.ParseCertificate(cert.Certificate[0]) + pool.AddCert(leaf) + cfg := &tls.Config{Certificates: []tls.Certificate{cert}, MinVersion: tls.VersionTLS12, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: pool} + ln, err := tls.Listen("tcp", "127.0.0.1:0", cfg) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = ln.Close() }) + go func() { + for { + c, err := ln.Accept() + if err != nil { + return + } + go func() { + defer c.Close() + if _, err := http.ReadRequest(bufio.NewReader(c)); err != nil { + return + } + _, _ = io.WriteString(c, "HTTP/1.1 404 Not Found\r\nContent-Length: 0\r\n\r\n") + }() + } + }() + return ln +} + +func TestSubstratePreflightRouter(t *testing.T) { + dir := t.TempDir() + bundle, ca, cert := selfSignedBundle(t, dir) + token := filepath.Join(dir, "token") + _ = os.WriteFile(token, []byte("t"), 0o600) + ln := fakeRouterListener(t, cert) + kube := &substrateKube{server: "http://127.0.0.1:1", tokenPath: token, caPath: ca, client: http.DefaultClient} + p := &SubstratePreflighter{kube: kube} + ctx := context.Background() + + // Good bundle, router verifying it → quiet. + opts := SubstrateOptions{RouterURL: "https://" + ln.Addr().String(), RouterClientBundle: bundle, RouterCAFile: ca} + if got := p.probeRouter(ctx, opts); len(got) != 0 { + t.Errorf("healthy router warned: %q", got) + } + + // Unusable bundle → credentials warning. + bad := opts + bad.RouterClientBundle = filepath.Join(dir, "missing.pem") + if got := p.probeRouter(ctx, bad); len(got) != 1 || !strings.Contains(got[0], "client credentials are not usable") { + t.Errorf("missing bundle: %q", got) + } + + // Router rejects the client certificate: a second CA the router does + // not trust. TLS 1.3 surfaces that on the first read, not the handshake. + otherDir := t.TempDir() + otherBundle, _, _ := selfSignedBundle(t, otherDir) + rejected := opts + rejected.RouterClientBundle = otherBundle + if got := p.probeRouter(ctx, rejected); len(got) != 1 || !strings.Contains(got[0], "rejected hive's client certificate") || !strings.Contains(got[0], otherBundle) { + t.Errorf("rejected cert: %q", got) + } + + // Unreachable router. + down := opts + down.RouterURL = "https://127.0.0.1:1" + if got := p.probeRouter(ctx, down); len(got) != 1 || !strings.Contains(got[0], "is unreachable") { + t.Errorf("unreachable: %q", got) + } + + // Injected dial wins over the real one; default bundle path is named + // when none is configured. + stub := &SubstratePreflighter{kube: kube, routerDial: func(context.Context, string, string) (net.Conn, error) { return nil, errors.New("stubbed") }} + if got := stub.probeRouter(ctx, opts); len(got) != 1 || !strings.Contains(got[0], "stubbed") { + t.Errorf("stubbed dial: %q", got) + } + if substrateRouterBundlePath(SubstrateOptions{}) != SubstrateDefaultRouterClientBundle { + t.Error("default bundle path") + } + + // Outside a cluster the router half is silent. + if got := (&SubstratePreflighter{}).probeRouter(ctx, opts); len(got) != 0 { + t.Errorf("outside cluster: %q", got) + } +} diff --git a/src/crux/sandbox/substrate_spec.go b/src/crux/sandbox/substrate_spec.go new file mode 100644 index 0000000000..d4df1e8a16 --- /dev/null +++ b/src/crux/sandbox/substrate_spec.go @@ -0,0 +1,234 @@ +package sandbox + +import ( + "errors" + "fmt" + "net/url" + "os" + "regexp" + "sort" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" +) + +// SubstrateDefaultBridgeCommand is the actor entrypoint when the operator +// names none: the bridge subcommand of the crux binary every hive sandbox +// image ships (ADR-0026 phase 2). +var SubstrateDefaultBridgeCommand = []string{"/usr/local/bin/crux", "substrate-bridge"} + +// SubstrateAlwaysAllowedHosts are the egress hosts every hive actor gets: +// the same GitHub hosts hive's MITM proxy (ADR-0002) brokers for a Podman +// sandbox and the OpenShell policy allows. Substrate's atenet-egress has no +// per-method read-only preset, so api.github.com is allowed as TLS +// passthrough like github.com; the actor still holds no token, and hive does +// every write itself after the kick. +var SubstrateAlwaysAllowedHosts = []string{"github.com", "api.github.com"} + +// substrateImageDigest accepts only digest-pinned references. +var substrateImageDigest = regexp.MustCompile(`^[^@\s]+@sha256:[0-9a-f]{64}$`) + +// actorTemplate maps the hive LaunchSpec onto a per-kick ActorTemplate: one +// container running the bridge, the workspace as a durableDir, golden +// snapshots off (a template used once has nothing to warm), and the +// operator's resource limits and pool selector. +func (l *SubstrateLauncher) actorTemplate(ref *substrateapi.ObjectRef, image string) *substrateapi.ActorTemplate { + command := l.Options.Command + if len(command) == 0 { + command = SubstrateDefaultBridgeCommand + } + cpu, mem := l.Options.CPUMillicores, l.Options.MemoryBytes + if cpu == 0 { + cpu = SubstrateDefaultCPUMillicores + } + if mem == 0 { + mem = SubstrateDefaultMemoryBytes + } + env := []*substrateapi.EnvVar{{Name: "HIVE_SUBSTRATE_BRIDGE_PORT", Value: strconv.Itoa(l.Options.bridgePort())}} + if p := strings.TrimSpace(l.Options.PolicyProfile); p != "" { + // Substrate has no annotations; the profile travels as an env var the + // bridge and in-actor tooling can read, and as a selector label an + // operator can pin a pool on. + env = append(env, &substrateapi.EnvVar{Name: "HIVE_POLICY_PROFILE", Value: p}) + } + return &substrateapi.ActorTemplate{ + Metadata: &substrateapi.ResourceMetadata{Atespace: ref.Atespace, Name: ref.Name}, + WorkerSelector: substrateSelector(l.Options.WorkerSelector), + Containers: []*substrateapi.Container{{ + Name: substrateContainer, + Image: image, + Command: command, + Env: env, + VolumeMounts: []*substrateapi.VolumeMount{{Name: substrateVolume, MountPath: SubstrateWorkspaceMount}}, + }}, + Volumes: []*substrateapi.Volume{{Name: substrateVolume, DurableDir: &substrateapi.DurableDirVolumeSource{}}}, + SnapshotConfig: &substrateapi.SnapshotConfig{GoldenSnapshotConfig: &substrateapi.GoldenSnapshotConfig{Mode: substrateapi.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED}}, + Resources: &substrateapi.ResourceRequirements{Limits: &substrateapi.Resources{CpuMillicores: cpu, MemoryBytes: mem}}, + } +} + +func substrateSelector(labels map[string]string) *substrateapi.Selector { + if len(labels) == 0 { + return nil + } + out := make(map[string]string, len(labels)) + for k, v := range labels { + out[k] = v + } + return &substrateapi.Selector{MatchLabels: out} +} + +// substrateEnv is the environment the kick runs with: the allowlisted, +// credential-filtered subset of hive's own, plus the spec's explicit values — +// the same rule the Podman, Job and OpenShell launchers apply. +func substrateEnv(spec LaunchSpec) map[string]string { + env := map[string]string{} + for _, entry := range SanitizedEnv(os.Environ(), spec.EnvAllowlist, spec.Env) { + if k, v, ok := strings.Cut(entry, "="); ok { + env[k] = v + } + } + return env +} + +// SubstrateEgressPolicy derives the actor's egress policy from hive's +// always-allowed GitHub hosts plus the operator's extra hosts, all as TLS +// passthrough on their port: the actor talks TLS end to end to the same hosts +// hive's MITM proxy would broker for a Podman sandbox, and atenet-egress +// default-denies everything else (including all UDP). Hosts are sorted so the +// policy hive sends is stable across kicks. +func SubstrateEgressPolicy(allowHosts []string) (*substrateapi.EgressPolicy, error) { + byPort := map[int32][]string{} + seen := map[string]bool{} + add := func(host string, port int32) { + key := host + ":" + strconv.Itoa(int(port)) + if seen[key] { + return + } + seen[key] = true + byPort[port] = append(byPort[port], host) + } + for _, h := range SubstrateAlwaysAllowedHosts { + add(h, substrateHTTPSPort) + } + for _, h := range allowHosts { + host, port, err := substrateHostPort(h) + if err != nil { + return nil, err + } + add(host, int32(port)) + } + ports := make([]int32, 0, len(byPort)) + for p := range byPort { + ports = append(ports, p) + } + sort.Slice(ports, func(i, j int) bool { return ports[i] < ports[j] }) + rules := make([]*substrateapi.EgressRule, 0, len(ports)) + for _, p := range ports { + hosts := byPort[p] + sort.Strings(hosts) + rules = append(rules, &substrateapi.EgressRule{TlsPassthrough: &substrateapi.TlsPassthroughRule{ + Hostnames: hosts, + Ports: &substrateapi.Ports{Numbers: []int32{p}}, + }}) + } + return &substrateapi.EgressPolicy{ + Metadata: &substrateapi.ResourceMetadata{Name: substrateEgressPolicy}, + Rules: rules, + }, nil +} + +// substrateAddress turns endpoint into a gRPC host:port target and whether +// TLS is implied. Accepts `host:port`, `http://host:port`, `https://host[:port]` +// and the grpc(s) spellings; a bare host:port is TLS, which is what the +// Control API serves. +func substrateAddress(raw string) (addr string, tls bool, err error) { + raw = strings.TrimSpace(raw) + if !strings.Contains(raw, "://") { + if _, _, err := substrateHostPort(raw); err != nil || !strings.Contains(raw, ":") { + return "", false, fmt.Errorf("substrate: endpoint %q must be host:port or an http(s) URL", raw) + } + return raw, true, nil + } + u, err := url.Parse(raw) + if err != nil || u.Host == "" { + return "", false, fmt.Errorf("substrate: endpoint %q is not a valid URL", raw) + } + switch u.Scheme { + case "https", "grpcs": + tls = true + case "http", "grpc": + default: + return "", false, fmt.Errorf("substrate: endpoint %q has unsupported scheme %q", raw, u.Scheme) + } + host := u.Host + if u.Port() == "" { + if tls { + host += ":443" + } else { + host += ":80" + } + } + return host, tls, nil +} + +// substrateRouterAddress validates router_url and returns its host:port; the +// router serves TLS only, so https is the one accepted scheme. +func substrateRouterAddress(raw string) (string, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.Host == "" || u.Scheme != "https" { + return "", fmt.Errorf("substrate: router_url %q must be an https URL (atenet-router serves TLS only)", raw) + } + host := u.Host + if u.Port() == "" { + host += ":443" + } + return host, nil +} + +// substrateHostName is a literal DNS name or a leftmost-wildcard DNS name, +// the two shapes Substrate's egress hostnames accept. +var substrateHostName = regexp.MustCompile(`^(\*\.)?[A-Za-z0-9]([A-Za-z0-9.-]*[A-Za-z0-9])?$`) + +// substrateHostPort parses an allow_hosts entry: `host` means 443. +func substrateHostPort(raw string) (string, uint32, error) { + raw = strings.TrimSpace(raw) + if raw == "" { + return "", 0, errors.New("substrate: allow_hosts entry is empty") + } + if strings.Contains(raw, "://") || strings.ContainsAny(raw, "/ ") { + return "", 0, fmt.Errorf("substrate: allow_hosts entry %q must be host or host:port, not a URL", raw) + } + host, portStr, found := strings.Cut(raw, ":") + if !substrateHostName.MatchString(host) { + return "", 0, fmt.Errorf("substrate: allow_hosts entry %q must be a DNS name with at most a leading *. wildcard — no userinfo, query or fragment", raw) + } + host = strings.ToLower(host) + if !found { + return host, substrateHTTPSPort, nil + } + port, err := strconv.ParseUint(portStr, 10, 16) + if err != nil || port == 0 { + return "", 0, fmt.Errorf("substrate: allow_hosts entry %q has an invalid port", raw) + } + return host, uint32(port), nil +} + +var substrateNameUnsafe = regexp.MustCompile(`[^a-z0-9-]+`) + +// substrateActorName builds a unique, DNS-label-safe actor (and template) +// name from the launch name and a timestamp: hive-kick--. +func substrateActorName(launchName string, now time.Time) string { + base := strings.Trim(substrateNameUnsafe.ReplaceAllString(strings.ToLower(strings.TrimSuffix(launchName, "-sandbox")), "-"), "-") + if base == "" { + base = "agent" + } + suffix := fmt.Sprintf("-%d", now.Unix()) + room := substrateNameMax - len(substrateNamePrefix) - len(suffix) + if len(base) > room { + base = strings.TrimRight(base[:room], "-") + } + return substrateNamePrefix + base + suffix +} diff --git a/src/crux/sandbox/substrate_test.go b/src/crux/sandbox/substrate_test.go new file mode 100644 index 0000000000..29bee045e4 --- /dev/null +++ b/src/crux/sandbox/substrate_test.go @@ -0,0 +1,767 @@ +package sandbox + +import ( + "bytes" + "context" + "errors" + "io" + "log/slog" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/sandbox/substrateapi" + "github.com/hivecommons/hive/crux/sandbox/substratebridge" +) + +// fakeSubstrate is an in-memory stand-in for the Control API slice and the +// in-actor bridge the launcher drives. It records the calls so tests can +// assert on the template, actor, policy and cleanup that reached the control +// plane, and lets each step fail or report a state on demand. +type fakeSubstrate struct { + mu sync.Mutex + + templates []*substrateapi.ActorTemplate + actors []*substrateapi.Actor + policies []*substrateapi.EgressPolicy + calls []string + templateErr error + actorErr error + policyErr error + resumeErr error + suspendErr error + deleteErr error + getErr error + states []substrateapi.ActorState + crashMsg string + spiffeID string + getCancels context.CancelFunc + waitReadyErr error + pushErr error + runErr error + runResp substratebridge.RunResponse + runCancels context.CancelFunc + pullErr error + runReq substratebridge.RunRequest + pushedWorkspace string + pulledWorkspace string + workers []*substrateapi.Worker + listErr error + deleted []string + closed bool + cleanupCtxDead bool +} + +func (f *fakeSubstrate) note(c string, ctx context.Context) { + f.mu.Lock() + defer f.mu.Unlock() + f.calls = append(f.calls, c) + if (c == "DeleteActor" || c == "DeleteActorTemplate" || c == "SuspendActor") && ctx.Err() != nil { + f.cleanupCtxDead = true + } +} + +func (f *fakeSubstrate) Control() substrateControl { return f } +func (f *fakeSubstrate) Bridge() substrateBridge { return f } +func (f *fakeSubstrate) Close() error { f.closed = true; return nil } + +func (f *fakeSubstrate) CreateActorTemplate(ctx context.Context, tpl *substrateapi.ActorTemplate) (*substrateapi.ActorTemplate, error) { + f.note("CreateActorTemplate", ctx) + if f.templateErr != nil { + return nil, f.templateErr + } + f.templates = append(f.templates, tpl) + return tpl, nil +} + +func (f *fakeSubstrate) DeleteActorTemplate(ctx context.Context, _ *substrateapi.ObjectRef) error { + f.note("DeleteActorTemplate", ctx) + return nil +} + +func (f *fakeSubstrate) CreateActor(ctx context.Context, a *substrateapi.Actor) (*substrateapi.Actor, error) { + f.note("CreateActor", ctx) + if f.actorErr != nil { + return nil, f.actorErr + } + f.actors = append(f.actors, a) + return a, nil +} + +func (f *fakeSubstrate) GetActor(ctx context.Context, ref *substrateapi.ObjectRef) (*substrateapi.Actor, error) { + f.note("GetActor", ctx) + if f.getErr != nil { + return nil, f.getErr + } + state := substrateapi.ActorState_ACTOR_STATE_RUNNING + if len(f.states) > 0 { + state, f.states = f.states[0], f.states[1:] + } + if f.getCancels != nil { + f.getCancels() + } + return &substrateapi.Actor{ + Metadata: &substrateapi.ResourceMetadata{Name: ref.GetName()}, + Status: &substrateapi.ActorStatus{ + State: state, Crash: &substrateapi.ActorCrash{Message: f.crashMsg}, + WorkerAssignment: &substrateapi.WorkerAssignment{WorkerSpiffeId: f.spiffeID}, + }, + }, nil +} + +func (f *fakeSubstrate) ResumeActor(ctx context.Context, _ *substrateapi.ObjectRef) error { + f.note("ResumeActor", ctx) + return f.resumeErr +} + +func (f *fakeSubstrate) SuspendActor(ctx context.Context, _ *substrateapi.ObjectRef, mode substrateapi.DurabilityMode) error { + f.note("SuspendActor", ctx) + f.calls = append(f.calls, "mode="+mode.String()) + return f.suspendErr +} + +func (f *fakeSubstrate) DeleteActor(ctx context.Context, ref *substrateapi.ObjectRef) error { + f.note("DeleteActor", ctx) + f.deleted = append(f.deleted, ref.GetName()) + return f.deleteErr +} + +func (f *fakeSubstrate) CreateActorEgressPolicy(ctx context.Context, _ *substrateapi.ObjectRef, p *substrateapi.EgressPolicy) error { + f.note("CreateActorEgressPolicy", ctx) + if f.policyErr != nil { + return f.policyErr + } + f.policies = append(f.policies, p) + return nil +} + +func (f *fakeSubstrate) ListWorkers(ctx context.Context) ([]*substrateapi.Worker, error) { + f.note("ListWorkers", ctx) + return f.workers, f.listErr +} + +func (f *fakeSubstrate) WaitReady(ctx context.Context, _ *substrateapi.ObjectRef, _ time.Duration) error { + f.note("WaitReady", ctx) + return f.waitReadyErr +} + +func (f *fakeSubstrate) Push(ctx context.Context, _ *substrateapi.ObjectRef, ws string) error { + f.note("Push", ctx) + f.pushedWorkspace = ws + return f.pushErr +} + +func (f *fakeSubstrate) Run(ctx context.Context, _ *substrateapi.ObjectRef, req substratebridge.RunRequest, live io.Writer) (substratebridge.RunResponse, error) { + f.note("Run", ctx) + f.runReq = req + if live != nil { + _, _ = io.WriteString(live, f.runResp.Stdout+f.runResp.Stderr) + } + if f.runCancels != nil { + f.runCancels() + } + return f.runResp, f.runErr +} + +func (f *fakeSubstrate) Pull(ctx context.Context, _ *substrateapi.ObjectRef, ws string) error { + f.note("Pull", ctx) + f.pulledWorkspace = ws + return f.pullErr +} + +func (f *fakeSubstrate) saw(call string) bool { + f.mu.Lock() + defer f.mu.Unlock() + for _, c := range f.calls { + if c == call { + return true + } + } + return false +} + +const substrateTestImage = "registry.example/hive/sandbox@sha256:0123456789abcdef0123456789abcdef0123456789abcdef0123456789abcdef" + +func newTestSubstrate(fake *fakeSubstrate) *SubstrateLauncher { + l := &SubstrateLauncher{Options: SubstrateOptions{ + Endpoint: "api.ate-system.svc:443", + Atespace: "hive", + Token: "tok", + Image: substrateTestImage, + AllowHosts: []string{"litellm.example.com", "*.pypi.org:8443"}, + }} + l.newClient = func(SubstrateOptions) (substrateClient, error) { return fake, nil } + l.now = func() time.Time { return time.Unix(1_700_000_000, 0) } + l.kicks = &SubstrateKicks{} + return l +} + +func substrateTestSpec() LaunchSpec { + return LaunchSpec{ + Image: "registry.example/hive/sandbox:dev", + Name: "deps-sandbox", + Workspace: "/data/agents/sandbox/deps", + Command: []string{"crux", "kick", "--agent", "deps"}, + Env: map[string]string{"HIVE_AGENT": "deps"}, + } +} + +func TestSubstrateRunHappyPath(t *testing.T) { + fake := &fakeSubstrate{runResp: substratebridge.RunResponse{Stdout: "done\n", Stderr: "warn\n", ExitCode: 0}} + l := newTestSubstrate(fake) + l.Options.PolicyProfile = "strict" + l.Options.WorkerSelector = map[string]string{"class": "gvisor"} + res, err := l.Run(context.Background(), substrateTestSpec()) + if err != nil { + t.Fatalf("Run: %v", err) + } + if res.Stdout != "done\n" || res.Stderr != "warn\n" || res.ExitCode != 0 { + t.Errorf("result = %+v", res) + } + want := []string{"CreateActorTemplate", "CreateActor", "CreateActorEgressPolicy", "ResumeActor", "GetActor", "WaitReady", "Push", "Run", "Pull", "DeleteActor", "DeleteActorTemplate"} + if got := strings.Join(fake.calls, " "); got != strings.Join(want, " ") { + t.Errorf("calls = %s\nwant = %s", got, strings.Join(want, " ")) + } + if !fake.closed { + t.Error("client not closed") + } + if len(fake.templates) != 1 { + t.Fatalf("templates = %d", len(fake.templates)) + } + tpl := fake.templates[0] + if tpl.GetMetadata().GetName() != "hive-kick-deps-1700000000" || tpl.GetMetadata().GetAtespace() != "hive" { + t.Errorf("template metadata = %v", tpl.GetMetadata()) + } + c := tpl.GetContainers()[0] + if c.GetImage() != substrateTestImage || strings.Join(c.GetCommand(), " ") != strings.Join(SubstrateDefaultBridgeCommand, " ") { + t.Errorf("container = %v", c) + } + var envNames []string + for _, e := range c.GetEnv() { + envNames = append(envNames, e.GetName()+"="+e.GetValue()) + } + if joined := strings.Join(envNames, ","); !strings.Contains(joined, "HIVE_SUBSTRATE_BRIDGE_PORT=8477") || !strings.Contains(joined, "HIVE_POLICY_PROFILE=strict") { + t.Errorf("env = %s", joined) + } + if tpl.GetWorkerSelector().GetMatchLabels()["class"] != "gvisor" || fake.actors[0].GetWorkerSelector().GetMatchLabels()["class"] != "gvisor" { + t.Error("worker selector not propagated to template and actor") + } + if tpl.GetVolumes()[0].GetDurableDir() == nil || c.GetVolumeMounts()[0].GetMountPath() != SubstrateWorkspaceMount { + t.Errorf("workspace volume = %v / %v", tpl.GetVolumes(), c.GetVolumeMounts()) + } + if tpl.GetSnapshotConfig().GetGoldenSnapshotConfig().GetMode() != substrateapi.GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED { + t.Error("golden snapshot not disabled") + } + if lim := tpl.GetResources().GetLimits(); lim.GetCpuMillicores() != SubstrateDefaultCPUMillicores || lim.GetMemoryBytes() != SubstrateDefaultMemoryBytes { + t.Errorf("limits = %v", lim) + } + if fake.actors[0].GetActorTemplate().GetName() != tpl.GetMetadata().GetName() { + t.Error("actor does not reference the per-kick template") + } + if fake.pushedWorkspace != "/data/agents/sandbox/deps" || fake.pulledWorkspace != "/data/agents/sandbox/deps" { + t.Errorf("workspace push/pull = %q / %q", fake.pushedWorkspace, fake.pulledWorkspace) + } + if fake.runReq.WorkDir != SubstrateWorkspaceMount || strings.Join(fake.runReq.Command, " ") != "crux kick --agent deps" || fake.runReq.Env["HIVE_AGENT"] != "deps" { + t.Errorf("run request = %+v", fake.runReq) + } + // Egress: GitHub hosts plus the operator's, grouped by port, sorted. + pol := fake.policies[0] + if pol.GetMetadata().GetName() != "default" || len(pol.GetRules()) != 2 { + t.Fatalf("policy = %v", pol) + } + r443 := pol.GetRules()[0].GetTlsPassthrough() + if strings.Join(r443.GetHostnames(), ",") != "api.github.com,github.com,litellm.example.com" || r443.GetPorts().GetNumbers()[0] != 443 { + t.Errorf("443 rule = %v", r443) + } + r8443 := pol.GetRules()[1].GetTlsPassthrough() + if strings.Join(r8443.GetHostnames(), ",") != "*.pypi.org" || r8443.GetPorts().GetNumbers()[0] != 8443 { + t.Errorf("8443 rule = %v", r8443) + } + if fake.cleanupCtxDead { + t.Error("cleanup ran on a dead context although the caller's was live") + } +} + +func TestSubstrateRunHonoursSpecOverrides(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.Image = "" + l.Options.Command = []string{"/opt/bridge"} + l.Options.CPUMillicores = 500 + l.Options.MemoryBytes = 1 << 30 + l.Options.BridgePort = 9000 + l.Options.Timeout = time.Minute + spec := substrateTestSpec() + spec.Image = substrateTestImage + spec.WorkDir = "/workspace/sub" + spec.NetworkNone = true + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("Run: %v", err) + } + c := fake.templates[0].GetContainers()[0] + if c.GetImage() != substrateTestImage || c.GetCommand()[0] != "/opt/bridge" || c.GetEnv()[0].GetValue() != "9000" { + t.Errorf("container = %v", c) + } + if lim := fake.templates[0].GetResources().GetLimits(); lim.GetCpuMillicores() != 500 || lim.GetMemoryBytes() != 1<<30 { + t.Errorf("limits = %v", lim) + } + if fake.templates[0].GetWorkerSelector() != nil { + t.Error("empty worker selector must be nil") + } + if len(fake.policies[0].GetRules()) != 0 { + t.Errorf("network_none must send an empty policy, got %v", fake.policies[0].GetRules()) + } + if fake.runReq.WorkDir != "/workspace/sub" { + t.Errorf("workdir = %q", fake.runReq.WorkDir) + } +} + +func TestSubstrateRunRefusesClineProvider(t *testing.T) { + spec, _ := clineLaunchFixture(t) + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + spec.Workspace = "/data/agents/sandbox/deps" + if _, err := l.Run(context.Background(), spec); err == nil || !strings.Contains(err.Error(), "Podman-only") { + t.Fatalf("err = %v", err) + } + if len(fake.calls) != 0 { + t.Error("control plane touched despite the Cline provider refusal") + } +} + +func TestSubstrateRunRejectsBadOptionsAndSpec(t *testing.T) { + cases := []struct { + name string + mut func(l *SubstrateLauncher, spec *LaunchSpec) + want string + }{ + {"bad endpoint", func(l *SubstrateLauncher, _ *LaunchSpec) { l.Options.Endpoint = "ftp://x" }, "unsupported scheme"}, + {"tag image in options", func(l *SubstrateLauncher, _ *LaunchSpec) { l.Options.Image = "img:latest" }, "pinned by digest"}, + {"no image anywhere", func(l *SubstrateLauncher, s *LaunchSpec) { l.Options.Image = ""; s.Image = "" }, "image is required"}, + {"tag image in spec", func(l *SubstrateLauncher, s *LaunchSpec) { l.Options.Image = ""; s.Image = "img:latest" }, "pinned by digest"}, + {"no workspace", func(_ *SubstrateLauncher, s *LaunchSpec) { s.Workspace = " " }, "workspace is required"}, + {"no command", func(_ *SubstrateLauncher, s *LaunchSpec) { s.Command = nil }, "command is required"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + spec := substrateTestSpec() + tc.mut(l, &spec) + _, err := l.Run(context.Background(), spec) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want %q", err, tc.want) + } + if len(fake.calls) != 0 { + t.Errorf("control plane touched: %v", fake.calls) + } + }) + } +} + +func TestSubstrateRunClientConstructorError(t *testing.T) { + l := newTestSubstrate(&fakeSubstrate{}) + l.newClient = func(SubstrateOptions) (substrateClient, error) { return nil, errors.New("no router bundle") } + if _, err := l.Run(context.Background(), substrateTestSpec()); err == nil || !strings.Contains(err.Error(), "no router bundle") { + t.Fatalf("err = %v", err) + } +} + +func TestSubstrateRunControlPlaneFailuresCleanUp(t *testing.T) { + cases := []struct { + name string + mut func(f *fakeSubstrate) + want string + cleanup []string + }{ + {"template", func(f *fakeSubstrate) { f.templateErr = errors.New("boom") }, "creating actor template", nil}, + {"actor", func(f *fakeSubstrate) { f.actorErr = errors.New("boom") }, "creating actor hive-kick", []string{"DeleteActorTemplate"}}, + {"policy", func(f *fakeSubstrate) { f.policyErr = errors.New("boom") }, "setting egress policy", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"resume", func(f *fakeSubstrate) { f.resumeErr = errors.New("boom") }, "resuming actor", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"get actor", func(f *fakeSubstrate) { f.getErr = errors.New("boom") }, "never became running", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"crashed", func(f *fakeSubstrate) { + f.states = []substrateapi.ActorState{substrateapi.ActorState_ACTOR_STATE_CRASHED} + f.crashMsg = "OOM" + }, "actor crashed: OOM", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"deleting", func(f *fakeSubstrate) { + f.states = []substrateapi.ActorState{substrateapi.ActorState_ACTOR_STATE_DELETING} + }, "entered ACTOR_STATE_DELETING", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"bridge never ready", func(f *fakeSubstrate) { f.waitReadyErr = errors.New("boom") }, "never answered", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"push", func(f *fakeSubstrate) { f.pushErr = errors.New("boom") }, "pushing workspace", []string{"DeleteActor", "DeleteActorTemplate"}}, + {"run transport", func(f *fakeSubstrate) { f.runErr = errors.New("boom") }, "running kick", []string{"Pull", "DeleteActor", "DeleteActorTemplate"}}, + {"run reported", func(f *fakeSubstrate) { + f.runResp = substratebridge.RunResponse{ExitCode: -1, Error: "exec: not found"} + }, "exec: not found", []string{"Pull", "DeleteActor", "DeleteActorTemplate"}}, + {"pull", func(f *fakeSubstrate) { f.pullErr = errors.New("boom") }, "pulling workspace", []string{"DeleteActor", "DeleteActorTemplate"}}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + fake := &fakeSubstrate{} + tc.mut(fake) + l := newTestSubstrate(fake) + _, err := l.Run(context.Background(), substrateTestSpec()) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want %q", err, tc.want) + } + for _, c := range tc.cleanup { + if !fake.saw(c) { + t.Errorf("cleanup %s not called: %v", c, fake.calls) + } + } + if tc.name == "template" && fake.saw("DeleteActorTemplate") { + t.Error("template deleted although it was never created") + } + if !fake.closed { + t.Error("client not closed") + } + }) + } +} + +func TestSubstrateRunReportsNonZeroExit(t *testing.T) { + fake := &fakeSubstrate{runResp: substratebridge.RunResponse{Stdout: "x", ExitCode: 3}} + res, err := newTestSubstrate(fake).Run(context.Background(), substrateTestSpec()) + if err != nil { + t.Fatalf("Run: %v (a non-zero exit is a result, not an error)", err) + } + if res.ExitCode != 3 || res.Stdout != "x" { + t.Errorf("result = %+v", res) + } +} + +func TestSubstrateRunSnapshotOnIdle(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + if !fake.saw("SuspendActor") || !fake.saw("mode=DURABILITY_MODE_ASYNCHRONOUS") || fake.saw("DeleteActor") { + t.Errorf("snapshot_on_idle must suspend asynchronously instead of deleting: %v", fake.calls) + } + if !fake.saw("DeleteActorTemplate") { + t.Error("per-kick template must still be removed") + } + + // A failed suspend falls back to delete so no actor is left running. + fake = &fakeSubstrate{suspendErr: errors.New("no snapshot store")} + l = newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + if !fake.saw("SuspendActor") || !fake.saw("DeleteActor") { + t.Errorf("suspend failure must fall back to delete: %v", fake.calls) + } +} + +func TestSubstrateSnapshotOnIdleDeletesSupersededActor(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + spec := substrateTestSpec() + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("Run: %v", err) + } + first := substrateActorName(spec.Name, l.now()) + if len(fake.deleted) != 0 { + t.Fatalf("first suspend deleted %v, want nothing", fake.deleted) + } + + l.now = func() time.Time { return time.Unix(1_700_003_600, 0) } + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("second Run: %v", err) + } + second := substrateActorName(spec.Name, l.now()) + if len(fake.deleted) != 1 || fake.deleted[0] != first || first == second { + t.Errorf("deleted = %v, want only the first actor %s (second %s survives)", fake.deleted, first, second) + } +} + +func TestSubstrateSnapshotOnIdleKeepsOtherAgentsActors(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + other := substrateTestSpec() + other.Name = "docs-sandbox" + if _, err := l.Run(context.Background(), other); err != nil { + t.Fatalf("Run other: %v", err) + } + l.now = func() time.Time { return time.Unix(1_700_003_600, 0) } + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + if len(fake.deleted) != 0 { + t.Errorf("deleted %v: one agent's suspend must not delete another's actor", fake.deleted) + } +} + +func TestSubstrateSnapshotOnIdleSkipsSupersededActorInOtherAtespace(t *testing.T) { + fake := &fakeSubstrate{} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + var logs bytes.Buffer + l.Options.Logger = slog.New(slog.NewTextHandler(&logs, nil)) + spec := substrateTestSpec() + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("Run: %v", err) + } + first := substrateActorName(spec.Name, l.now()) + + // The same agent is reconfigured onto another atespace: the launcher's + // client is for the new one, so the old actor must not be deleted + // through it (the delete would target a name never created there). + l.Options.Atespace = "hive-staging" + l.now = func() time.Time { return time.Unix(1_700_003_600, 0) } + if _, err := l.Run(context.Background(), spec); err != nil { + t.Fatalf("second Run: %v", err) + } + if len(fake.deleted) != 0 { + t.Errorf("deleted = %v, want no delete across atespaces", fake.deleted) + } + if out := logs.String(); !strings.Contains(out, first) || !strings.Contains(out, "another atespace") || !strings.Contains(out, "agent="+spec.Name) { + t.Errorf("log = %q, want a warning naming %s and the agent", out, first) + } +} + +func TestSubstrateSnapshotOnIdleDeleteErrorLogsStructured(t *testing.T) { + fake := &fakeSubstrate{deleteErr: errors.New("not found")} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + var logs bytes.Buffer + l.Options.Logger = slog.New(slog.NewTextHandler(&logs, nil)) + for _, unix := range []int64{1_700_000_000, 1_700_003_600} { + l.now = func() time.Time { return time.Unix(unix, 0) } + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v", err) + } + } + out := logs.String() + if !strings.Contains(out, "level=WARN") || !strings.Contains(out, "agent=deps-sandbox") || !strings.Contains(out, "error=\"not found\"") { + t.Errorf("log = %q, want a WARN record with agent and error attrs", out) + } +} + +func TestSubstrateSnapshotOnIdleDeleteErrorDoesNotFailRun(t *testing.T) { + fake := &fakeSubstrate{deleteErr: errors.New("not found")} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true + for _, unix := range []int64{1_700_000_000, 1_700_003_600} { + l.now = func() time.Time { return time.Unix(unix, 0) } + if _, err := l.Run(context.Background(), substrateTestSpec()); err != nil { + t.Fatalf("Run: %v (a failed GC delete must not fail the kick)", err) + } + } + if len(fake.deleted) != 1 { + t.Errorf("deleted = %v, want one attempted delete", fake.deleted) + } +} + +func TestSubstrateRunContextCancelledMidKick(t *testing.T) { + // The kick's context dies while the command runs: Pull is skipped, the + // actor is deleted on a fresh context, and the caller gets ctx.Err(). + ctx, cancel := context.WithCancel(context.Background()) + fake := &fakeSubstrate{runCancels: cancel} + l := newTestSubstrate(fake) + l.Options.SnapshotOnIdle = true // must not snapshot on a dead context + _, err := l.Run(ctx, substrateTestSpec()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if fake.saw("Pull") || fake.saw("SuspendActor") { + t.Errorf("pull or suspend ran after cancel: %v", fake.calls) + } + if !fake.saw("DeleteActor") || !fake.saw("DeleteActorTemplate") { + t.Errorf("cleanup skipped: %v", fake.calls) + } + if fake.cleanupCtxDead { + t.Error("cleanup used the dead caller context instead of a fresh one") + } +} + +func TestSubstrateRunContextCancelledWhileWaitingForRunning(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + fake := &fakeSubstrate{getCancels: cancel, states: []substrateapi.ActorState{substrateapi.ActorState_ACTOR_STATE_RESUMING}} + _, err := newTestSubstrate(fake).Run(ctx, substrateTestSpec()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if !fake.saw("DeleteActor") || fake.saw("WaitReady") { + t.Errorf("calls = %v", fake.calls) + } +} + +func TestSubstrateRunContextCancelledWhileWaitingForBridge(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + fake := &fakeSubstrate{waitReadyErr: context.Canceled} + l := newTestSubstrate(fake) + // Cancel just before WaitReady by hooking GetActor (the step before). + fake.getCancels = cancel + _, err := l.Run(ctx, substrateTestSpec()) + if !errors.Is(err, context.Canceled) { + t.Fatalf("err = %v, want context.Canceled", err) + } + if fake.saw("Push") { + t.Errorf("push ran after cancel: %v", fake.calls) + } +} + +func TestSubstrateWaitRunningPollsUntilRunning(t *testing.T) { + fake := &fakeSubstrate{states: []substrateapi.ActorState{ + substrateapi.ActorState_ACTOR_STATE_SUSPENDED, + substrateapi.ActorState_ACTOR_STATE_RESUMING, + substrateapi.ActorState_ACTOR_STATE_RUNNING, + }} + ref := &substrateapi.ObjectRef{Atespace: "hive", Name: "a"} + if err := substrateWaitRunning(context.Background(), fake, ref, time.Millisecond, nil); err != nil { + t.Fatalf("err = %v", err) + } + if n := len(fake.calls); n != 3 { + t.Errorf("GetActor calls = %d, want 3", n) + } + fake = &fakeSubstrate{states: []substrateapi.ActorState{substrateapi.ActorState_ACTOR_STATE_SUSPENDING}} + if err := substrateWaitRunning(context.Background(), fake, ref, time.Millisecond, nil); err == nil || !strings.Contains(err.Error(), "SUSPENDING") { + t.Errorf("err = %v", err) + } +} + +func TestSubstrateOptionsValidate(t *testing.T) { + good := SubstrateOptions{Image: substrateTestImage, AllowHosts: []string{"a.example.com", "*.b.example.com:8443"}} + if err := good.Validate(); err != nil { + t.Fatalf("Validate(good) = %v", err) + } + if err := (SubstrateOptions{}).Validate(); err != nil { + t.Fatalf("Validate(defaults) = %v", err) + } + cases := map[string]SubstrateOptions{ + "endpoint no port": {Endpoint: "api.ate-system.svc"}, + "endpoint bad url": {Endpoint: "https://"}, + "endpoint scheme": {Endpoint: "ftp://x:1"}, + "router http": {RouterURL: "http://router:8444"}, + "router garbage": {RouterURL: "::not a url"}, + "image tag": {Image: "img:latest"}, + "negative cpu": {CPUMillicores: -1}, + "negative mem": {MemoryBytes: -1}, + "bridge port range": {BridgePort: 70000}, + "allow host url": {AllowHosts: []string{"https://x.example.com"}}, + "allow host empty": {AllowHosts: []string{" "}}, + "allow host userinfo": {AllowHosts: []string{"u@x.example.com"}}, + "allow host bad port": {AllowHosts: []string{"x.example.com:0"}}, + "allow host port alpha": {AllowHosts: []string{"x.example.com:abc"}}, + "allow host wildcard": {AllowHosts: []string{"*x.example.com"}}, + } + for name, o := range cases { + if err := o.Validate(); err == nil { + t.Errorf("%s: Validate accepted %+v", name, o) + } + } +} + +func TestSubstrateAddressForms(t *testing.T) { + cases := []struct { + in string + addr string + tls bool + }{ + {"api.ate-system.svc:443", "api.ate-system.svc:443", true}, + {"https://api.ate-system.svc", "api.ate-system.svc:443", true}, + {"grpcs://api:9443", "api:9443", true}, + {"http://localhost:8080", "localhost:8080", false}, + {"http://localhost", "localhost:80", false}, + {"grpc://api:50051", "api:50051", false}, + } + for _, tc := range cases { + addr, tls, err := substrateAddress(tc.in) + if err != nil || addr != tc.addr || tls != tc.tls { + t.Errorf("substrateAddress(%q) = %q %v %v", tc.in, addr, tls, err) + } + } + if got, err := substrateRouterAddress("https://atenet-router.ate-system.svc"); err != nil || got != "atenet-router.ate-system.svc:443" { + t.Errorf("router default port: %q %v", got, err) + } + if got, err := substrateRouterAddress("https://r:8444"); err != nil || got != "r:8444" { + t.Errorf("router explicit port: %q %v", got, err) + } +} + +func TestSubstrateHostPort(t *testing.T) { + host, port, err := substrateHostPort("API.Example.COM") + if err != nil || host != "api.example.com" || port != 443 { + t.Errorf("got %q %d %v", host, port, err) + } + host, port, err = substrateHostPort("*.pypi.org:8443") + if err != nil || host != "*.pypi.org" || port != 8443 { + t.Errorf("got %q %d %v", host, port, err) + } + for _, bad := range []string{"", "a b", "x/y", "x.example.com:99999", "-bad.example.com", "x.example.com:"} { + if _, _, err := substrateHostPort(bad); err == nil { + t.Errorf("accepted %q", bad) + } + } +} + +func TestSubstrateEgressPolicyDedupesAndErrors(t *testing.T) { + pol, err := SubstrateEgressPolicy([]string{"github.com", "GitHub.com:443", "x.example.com:8443"}) + if err != nil { + t.Fatal(err) + } + if len(pol.GetRules()) != 2 || len(pol.GetRules()[0].GetTlsPassthrough().GetHostnames()) != 2 { + t.Errorf("policy = %v", pol) + } + if _, err := SubstrateEgressPolicy([]string{"bad host"}); err == nil { + t.Error("bad host accepted") + } +} + +func TestSubstrateActorName(t *testing.T) { + at := time.Unix(1_700_000_000, 0) + if got := substrateActorName("Deps_Review-sandbox", at); got != "hive-kick-deps-review-1700000000" { + t.Errorf("got %q", got) + } + if got := substrateActorName("---", at); got != "hive-kick-agent-1700000000" { + t.Errorf("empty base: %q", got) + } + long := substrateActorName(strings.Repeat("a", 80)+"-b", at) + if len(long) > substrateNameMax || !strings.HasPrefix(long, "hive-kick-aaaa") || !strings.HasSuffix(long, "-1700000000") { + t.Errorf("long name = %q (%d)", long, len(long)) + } + trimmed := substrateActorName(strings.Repeat("a", 41)+"-"+strings.Repeat("b", 20), at) + if strings.Contains(trimmed, "--") { + t.Errorf("truncation left a dangling dash: %q", trimmed) + } +} + +func TestSubstrateOptionDefaults(t *testing.T) { + o := SubstrateOptions{} + if o.endpoint() != SubstrateDefaultEndpoint || o.atespace() != SubstrateDefaultAtespace || o.routerURL() != SubstrateDefaultRouterURL || o.bridgePort() != substratebridge.DefaultPort || o.tokenFile() != SubstrateDefaultTokenFile { + t.Errorf("defaults = %q %q %q %d %q", o.endpoint(), o.atespace(), o.routerURL(), o.bridgePort(), o.tokenFile()) + } + o = SubstrateOptions{Endpoint: " e:1 ", Atespace: " a ", RouterURL: " https://r ", BridgePort: 1, TokenFile: " /t "} + if o.endpoint() != "e:1" || o.atespace() != "a" || o.routerURL() != "https://r" || o.bridgePort() != 1 || o.tokenFile() != "/t" { + t.Errorf("trimmed = %q %q %q %d %q", o.endpoint(), o.atespace(), o.routerURL(), o.bridgePort(), o.tokenFile()) + } +} + +func TestSubstrateEnvFiltersCredentials(t *testing.T) { + t.Setenv("GITHUB_TOKEN", "ghp_secret") + t.Setenv("HIVE_SUBSTRATE_TEST_KEEP", "yes") + spec := substrateTestSpec() + spec.EnvAllowlist = []string{"HIVE_SUBSTRATE_TEST_KEEP"} + env := substrateEnv(spec) + if env["HIVE_SUBSTRATE_TEST_KEEP"] != "yes" || env["HIVE_AGENT"] != "deps" { + t.Errorf("allowlisted values missing: %v", env) + } + if _, ok := env["GITHUB_TOKEN"]; ok { + t.Error("GITHUB_TOKEN leaked into the actor env") + } +} + +func TestSubstrateLauncherDefaultClientIsGRPC(t *testing.T) { + // Without a test hook the launcher builds the real client; with no + // router bundle on disk that fails before any network I/O. + l := &SubstrateLauncher{Options: SubstrateOptions{InsecureTLS: true, RouterClientBundle: "/nonexistent/bundle.pem"}} + if _, err := l.client(); err == nil || !strings.Contains(err.Error(), "router_client_bundle") { + t.Fatalf("client() = %v", err) + } +} diff --git a/src/crux/sandbox/substrateapi/actor.pb.go b/src/crux/sandbox/substrateapi/actor.pb.go new file mode 100644 index 0000000000..d70548216f --- /dev/null +++ b/src/crux/sandbox/substrateapi/actor.pb.go @@ -0,0 +1,586 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: actor.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type ActorState int32 + +const ( + ActorState_ACTOR_STATE_UNSPECIFIED ActorState = 0 + ActorState_ACTOR_STATE_RESUMING ActorState = 1 + ActorState_ACTOR_STATE_RUNNING ActorState = 2 + ActorState_ACTOR_STATE_SUSPENDING ActorState = 3 + ActorState_ACTOR_STATE_SUSPENDED ActorState = 4 + ActorState_ACTOR_STATE_CRASHED ActorState = 5 + ActorState_ACTOR_STATE_DELETING ActorState = 6 + ActorState_ACTOR_STATE_REVERTING ActorState = 7 +) + +// Enum value maps for ActorState. +var ( + ActorState_name = map[int32]string{ + 0: "ACTOR_STATE_UNSPECIFIED", + 1: "ACTOR_STATE_RESUMING", + 2: "ACTOR_STATE_RUNNING", + 3: "ACTOR_STATE_SUSPENDING", + 4: "ACTOR_STATE_SUSPENDED", + 5: "ACTOR_STATE_CRASHED", + 6: "ACTOR_STATE_DELETING", + 7: "ACTOR_STATE_REVERTING", + } + ActorState_value = map[string]int32{ + "ACTOR_STATE_UNSPECIFIED": 0, + "ACTOR_STATE_RESUMING": 1, + "ACTOR_STATE_RUNNING": 2, + "ACTOR_STATE_SUSPENDING": 3, + "ACTOR_STATE_SUSPENDED": 4, + "ACTOR_STATE_CRASHED": 5, + "ACTOR_STATE_DELETING": 6, + "ACTOR_STATE_REVERTING": 7, + } +) + +func (x ActorState) Enum() *ActorState { + p := new(ActorState) + *p = x + return p +} + +func (x ActorState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (ActorState) Descriptor() protoreflect.EnumDescriptor { + return file_actor_proto_enumTypes[0].Descriptor() +} + +func (ActorState) Type() protoreflect.EnumType { + return &file_actor_proto_enumTypes[0] +} + +func (x ActorState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use ActorState.Descriptor instead. +func (ActorState) EnumDescriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{0} +} + +type Actor struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metadata *ResourceMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + ActorTemplate *ObjectRef `protobuf:"bytes,2,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` + WorkerSelector *Selector `protobuf:"bytes,3,opt,name=worker_selector,json=workerSelector,proto3" json:"worker_selector,omitempty"` + SourceTag *ObjectRef `protobuf:"bytes,4,opt,name=source_tag,json=sourceTag,proto3" json:"source_tag,omitempty"` + Status *ActorStatus `protobuf:"bytes,5,opt,name=status,proto3" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Actor) Reset() { + *x = Actor{} + mi := &file_actor_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Actor) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Actor) ProtoMessage() {} + +func (x *Actor) ProtoReflect() protoreflect.Message { + mi := &file_actor_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Actor.ProtoReflect.Descriptor instead. +func (*Actor) Descriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{0} +} + +func (x *Actor) GetMetadata() *ResourceMetadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *Actor) GetActorTemplate() *ObjectRef { + if x != nil { + return x.ActorTemplate + } + return nil +} + +func (x *Actor) GetWorkerSelector() *Selector { + if x != nil { + return x.WorkerSelector + } + return nil +} + +func (x *Actor) GetSourceTag() *ObjectRef { + if x != nil { + return x.SourceTag + } + return nil +} + +func (x *Actor) GetStatus() *ActorStatus { + if x != nil { + return x.Status + } + return nil +} + +type ActorStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + State ActorState `protobuf:"varint,1,opt,name=state,proto3,enum=substrate.ateapi.v1.ActorState" json:"state,omitempty"` + WorkerAssignment *WorkerAssignment `protobuf:"bytes,2,opt,name=worker_assignment,json=workerAssignment,proto3" json:"worker_assignment,omitempty"` + Crash *ActorCrash `protobuf:"bytes,4,opt,name=crash,proto3" json:"crash,omitempty"` + AssignedNode string `protobuf:"bytes,5,opt,name=assigned_node,json=assignedNode,proto3" json:"assigned_node,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActorStatus) Reset() { + *x = ActorStatus{} + mi := &file_actor_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActorStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActorStatus) ProtoMessage() {} + +func (x *ActorStatus) ProtoReflect() protoreflect.Message { + mi := &file_actor_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActorStatus.ProtoReflect.Descriptor instead. +func (*ActorStatus) Descriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{1} +} + +func (x *ActorStatus) GetState() ActorState { + if x != nil { + return x.State + } + return ActorState_ACTOR_STATE_UNSPECIFIED +} + +func (x *ActorStatus) GetWorkerAssignment() *WorkerAssignment { + if x != nil { + return x.WorkerAssignment + } + return nil +} + +func (x *ActorStatus) GetCrash() *ActorCrash { + if x != nil { + return x.Crash + } + return nil +} + +func (x *ActorStatus) GetAssignedNode() string { + if x != nil { + return x.AssignedNode + } + return "" +} + +type ActorCrash struct { + state protoimpl.MessageState `protogen:"open.v1"` + Message string `protobuf:"bytes,1,opt,name=message,proto3" json:"message,omitempty"` + CrashTime *timestamppb.Timestamp `protobuf:"bytes,2,opt,name=crash_time,json=crashTime,proto3" json:"crash_time,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActorCrash) Reset() { + *x = ActorCrash{} + mi := &file_actor_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActorCrash) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActorCrash) ProtoMessage() {} + +func (x *ActorCrash) ProtoReflect() protoreflect.Message { + mi := &file_actor_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActorCrash.ProtoReflect.Descriptor instead. +func (*ActorCrash) Descriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{2} +} + +func (x *ActorCrash) GetMessage() string { + if x != nil { + return x.Message + } + return "" +} + +func (x *ActorCrash) GetCrashTime() *timestamppb.Timestamp { + if x != nil { + return x.CrashTime + } + return nil +} + +type WorkerAssignment struct { + state protoimpl.MessageState `protogen:"open.v1"` + Worker *ObjectRef `protobuf:"bytes,6,opt,name=worker,proto3" json:"worker,omitempty"` + WorkerNamespace string `protobuf:"bytes,1,opt,name=worker_namespace,json=workerNamespace,proto3" json:"worker_namespace,omitempty"` + WorkerPool string `protobuf:"bytes,2,opt,name=worker_pool,json=workerPool,proto3" json:"worker_pool,omitempty"` + WorkerPod string `protobuf:"bytes,3,opt,name=worker_pod,json=workerPod,proto3" json:"worker_pod,omitempty"` + WorkerPodUid string `protobuf:"bytes,4,opt,name=worker_pod_uid,json=workerPodUid,proto3" json:"worker_pod_uid,omitempty"` + WorkerPodIps []string `protobuf:"bytes,5,rep,name=worker_pod_ips,json=workerPodIps,proto3" json:"worker_pod_ips,omitempty"` + NodeName string `protobuf:"bytes,7,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"` + WorkerEpoch int64 `protobuf:"varint,8,opt,name=worker_epoch,json=workerEpoch,proto3" json:"worker_epoch,omitempty"` + SandboxRuntime *ActorSandboxRuntime `protobuf:"bytes,9,opt,name=sandbox_runtime,json=sandboxRuntime,proto3" json:"sandbox_runtime,omitempty"` + WorkerSpiffeId string `protobuf:"bytes,10,opt,name=worker_spiffe_id,json=workerSpiffeId,proto3" json:"worker_spiffe_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WorkerAssignment) Reset() { + *x = WorkerAssignment{} + mi := &file_actor_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WorkerAssignment) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WorkerAssignment) ProtoMessage() {} + +func (x *WorkerAssignment) ProtoReflect() protoreflect.Message { + mi := &file_actor_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WorkerAssignment.ProtoReflect.Descriptor instead. +func (*WorkerAssignment) Descriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{3} +} + +func (x *WorkerAssignment) GetWorker() *ObjectRef { + if x != nil { + return x.Worker + } + return nil +} + +func (x *WorkerAssignment) GetWorkerNamespace() string { + if x != nil { + return x.WorkerNamespace + } + return "" +} + +func (x *WorkerAssignment) GetWorkerPool() string { + if x != nil { + return x.WorkerPool + } + return "" +} + +func (x *WorkerAssignment) GetWorkerPod() string { + if x != nil { + return x.WorkerPod + } + return "" +} + +func (x *WorkerAssignment) GetWorkerPodUid() string { + if x != nil { + return x.WorkerPodUid + } + return "" +} + +func (x *WorkerAssignment) GetWorkerPodIps() []string { + if x != nil { + return x.WorkerPodIps + } + return nil +} + +func (x *WorkerAssignment) GetNodeName() string { + if x != nil { + return x.NodeName + } + return "" +} + +func (x *WorkerAssignment) GetWorkerEpoch() int64 { + if x != nil { + return x.WorkerEpoch + } + return 0 +} + +func (x *WorkerAssignment) GetSandboxRuntime() *ActorSandboxRuntime { + if x != nil { + return x.SandboxRuntime + } + return nil +} + +func (x *WorkerAssignment) GetWorkerSpiffeId() string { + if x != nil { + return x.WorkerSpiffeId + } + return "" +} + +type ActorSandboxRuntime struct { + state protoimpl.MessageState `protogen:"open.v1"` + SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActorSandboxRuntime) Reset() { + *x = ActorSandboxRuntime{} + mi := &file_actor_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActorSandboxRuntime) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActorSandboxRuntime) ProtoMessage() {} + +func (x *ActorSandboxRuntime) ProtoReflect() protoreflect.Message { + mi := &file_actor_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActorSandboxRuntime.ProtoReflect.Descriptor instead. +func (*ActorSandboxRuntime) Descriptor() ([]byte, []int) { + return file_actor_proto_rawDescGZIP(), []int{4} +} + +func (x *ActorSandboxRuntime) GetSandboxClass() string { + if x != nil { + return x.SandboxClass + } + return "" +} + +var File_actor_proto protoreflect.FileDescriptor + +const file_actor_proto_rawDesc = "" + + "\n" + + "\vactor.proto\x12\x13substrate.ateapi.v1\x1a\fcommon.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"\xd2\x02\n" + + "\x05Actor\x12A\n" + + "\bmetadata\x18\x01 \x01(\v2%.substrate.ateapi.v1.ResourceMetadataR\bmetadata\x12E\n" + + "\x0eactor_template\x18\x02 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\ractorTemplate\x12F\n" + + "\x0fworker_selector\x18\x03 \x01(\v2\x1d.substrate.ateapi.v1.SelectorR\x0eworkerSelector\x12=\n" + + "\n" + + "source_tag\x18\x04 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\tsourceTag\x128\n" + + "\x06status\x18\x05 \x01(\v2 .substrate.ateapi.v1.ActorStatusR\x06status\"\xf4\x01\n" + + "\vActorStatus\x125\n" + + "\x05state\x18\x01 \x01(\x0e2\x1f.substrate.ateapi.v1.ActorStateR\x05state\x12R\n" + + "\x11worker_assignment\x18\x02 \x01(\v2%.substrate.ateapi.v1.WorkerAssignmentR\x10workerAssignment\x125\n" + + "\x05crash\x18\x04 \x01(\v2\x1f.substrate.ateapi.v1.ActorCrashR\x05crash\x12#\n" + + "\rassigned_node\x18\x05 \x01(\tR\fassignedNode\"a\n" + + "\n" + + "ActorCrash\x12\x18\n" + + "\amessage\x18\x01 \x01(\tR\amessage\x129\n" + + "\n" + + "crash_time\x18\x02 \x01(\v2\x1a.google.protobuf.TimestampR\tcrashTime\"\xbe\x03\n" + + "\x10WorkerAssignment\x126\n" + + "\x06worker\x18\x06 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x06worker\x12)\n" + + "\x10worker_namespace\x18\x01 \x01(\tR\x0fworkerNamespace\x12\x1f\n" + + "\vworker_pool\x18\x02 \x01(\tR\n" + + "workerPool\x12\x1d\n" + + "\n" + + "worker_pod\x18\x03 \x01(\tR\tworkerPod\x12$\n" + + "\x0eworker_pod_uid\x18\x04 \x01(\tR\fworkerPodUid\x12$\n" + + "\x0eworker_pod_ips\x18\x05 \x03(\tR\fworkerPodIps\x12\x1b\n" + + "\tnode_name\x18\a \x01(\tR\bnodeName\x12!\n" + + "\fworker_epoch\x18\b \x01(\x03R\vworkerEpoch\x12Q\n" + + "\x0fsandbox_runtime\x18\t \x01(\v2(.substrate.ateapi.v1.ActorSandboxRuntimeR\x0esandboxRuntime\x12(\n" + + "\x10worker_spiffe_id\x18\n" + + " \x01(\tR\x0eworkerSpiffeId\":\n" + + "\x13ActorSandboxRuntime\x12#\n" + + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass*\xe1\x01\n" + + "\n" + + "ActorState\x12\x1b\n" + + "\x17ACTOR_STATE_UNSPECIFIED\x10\x00\x12\x18\n" + + "\x14ACTOR_STATE_RESUMING\x10\x01\x12\x17\n" + + "\x13ACTOR_STATE_RUNNING\x10\x02\x12\x1a\n" + + "\x16ACTOR_STATE_SUSPENDING\x10\x03\x12\x19\n" + + "\x15ACTOR_STATE_SUSPENDED\x10\x04\x12\x17\n" + + "\x13ACTOR_STATE_CRASHED\x10\x05\x12\x18\n" + + "\x14ACTOR_STATE_DELETING\x10\x06\x12\x19\n" + + "\x15ACTOR_STATE_REVERTING\x10\aBDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_actor_proto_rawDescOnce sync.Once + file_actor_proto_rawDescData []byte +) + +func file_actor_proto_rawDescGZIP() []byte { + file_actor_proto_rawDescOnce.Do(func() { + file_actor_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_actor_proto_rawDesc), len(file_actor_proto_rawDesc))) + }) + return file_actor_proto_rawDescData +} + +var file_actor_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_actor_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_actor_proto_goTypes = []any{ + (ActorState)(0), // 0: substrate.ateapi.v1.ActorState + (*Actor)(nil), // 1: substrate.ateapi.v1.Actor + (*ActorStatus)(nil), // 2: substrate.ateapi.v1.ActorStatus + (*ActorCrash)(nil), // 3: substrate.ateapi.v1.ActorCrash + (*WorkerAssignment)(nil), // 4: substrate.ateapi.v1.WorkerAssignment + (*ActorSandboxRuntime)(nil), // 5: substrate.ateapi.v1.ActorSandboxRuntime + (*ResourceMetadata)(nil), // 6: substrate.ateapi.v1.ResourceMetadata + (*ObjectRef)(nil), // 7: substrate.ateapi.v1.ObjectRef + (*Selector)(nil), // 8: substrate.ateapi.v1.Selector + (*timestamppb.Timestamp)(nil), // 9: google.protobuf.Timestamp +} +var file_actor_proto_depIdxs = []int32{ + 6, // 0: substrate.ateapi.v1.Actor.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 7, // 1: substrate.ateapi.v1.Actor.actor_template:type_name -> substrate.ateapi.v1.ObjectRef + 8, // 2: substrate.ateapi.v1.Actor.worker_selector:type_name -> substrate.ateapi.v1.Selector + 7, // 3: substrate.ateapi.v1.Actor.source_tag:type_name -> substrate.ateapi.v1.ObjectRef + 2, // 4: substrate.ateapi.v1.Actor.status:type_name -> substrate.ateapi.v1.ActorStatus + 0, // 5: substrate.ateapi.v1.ActorStatus.state:type_name -> substrate.ateapi.v1.ActorState + 4, // 6: substrate.ateapi.v1.ActorStatus.worker_assignment:type_name -> substrate.ateapi.v1.WorkerAssignment + 3, // 7: substrate.ateapi.v1.ActorStatus.crash:type_name -> substrate.ateapi.v1.ActorCrash + 9, // 8: substrate.ateapi.v1.ActorCrash.crash_time:type_name -> google.protobuf.Timestamp + 7, // 9: substrate.ateapi.v1.WorkerAssignment.worker:type_name -> substrate.ateapi.v1.ObjectRef + 5, // 10: substrate.ateapi.v1.WorkerAssignment.sandbox_runtime:type_name -> substrate.ateapi.v1.ActorSandboxRuntime + 11, // [11:11] is the sub-list for method output_type + 11, // [11:11] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name +} + +func init() { file_actor_proto_init() } +func file_actor_proto_init() { + if File_actor_proto != nil { + return + } + file_common_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_actor_proto_rawDesc), len(file_actor_proto_rawDesc)), + NumEnums: 1, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_actor_proto_goTypes, + DependencyIndexes: file_actor_proto_depIdxs, + EnumInfos: file_actor_proto_enumTypes, + MessageInfos: file_actor_proto_msgTypes, + }.Build() + File_actor_proto = out.File + file_actor_proto_goTypes = nil + file_actor_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/actor.proto b/src/crux/sandbox/substrateapi/actor.proto new file mode 100644 index 0000000000..d5df52af03 --- /dev/null +++ b/src/crux/sandbox/substrateapi/actor.proto @@ -0,0 +1,79 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "common.proto"; +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message Actor { + ResourceMetadata metadata = 1; + ObjectRef actor_template = 2; + Selector worker_selector = 3; + ObjectRef source_tag = 4; + ActorStatus status = 5; +} + +enum ActorState { + ACTOR_STATE_UNSPECIFIED = 0; + ACTOR_STATE_RESUMING = 1; + ACTOR_STATE_RUNNING = 2; + ACTOR_STATE_SUSPENDING = 3; + ACTOR_STATE_SUSPENDED = 4; + ACTOR_STATE_CRASHED = 5; + ACTOR_STATE_DELETING = 6; + ACTOR_STATE_REVERTING = 7; +} + +message ActorStatus { + ActorState state = 1; + WorkerAssignment worker_assignment = 2; + ActorCrash crash = 4; + string assigned_node = 5; +} + +message ActorCrash { + string message = 1; + google.protobuf.Timestamp crash_time = 2; +} + +message WorkerAssignment { + ObjectRef worker = 6; + string worker_namespace = 1; + string worker_pool = 2; + string worker_pod = 3; + string worker_pod_uid = 4; + repeated string worker_pod_ips = 5; + string node_name = 7; + int64 worker_epoch = 8; + ActorSandboxRuntime sandbox_runtime = 9; + string worker_spiffe_id = 10; +} + +message ActorSandboxRuntime { + string sandbox_class = 1; +} diff --git a/src/crux/sandbox/substrateapi/actor_rpc.pb.go b/src/crux/sandbox/substrateapi/actor_rpc.pb.go new file mode 100644 index 0000000000..17d81ba51d --- /dev/null +++ b/src/crux/sandbox/substrateapi/actor_rpc.pb.go @@ -0,0 +1,572 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: actor_rpc.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type DurabilityMode int32 + +const ( + DurabilityMode_DURABILITY_MODE_UNSPECIFIED DurabilityMode = 0 + DurabilityMode_DURABILITY_MODE_SYNCHRONOUS DurabilityMode = 1 + DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS DurabilityMode = 2 +) + +// Enum value maps for DurabilityMode. +var ( + DurabilityMode_name = map[int32]string{ + 0: "DURABILITY_MODE_UNSPECIFIED", + 1: "DURABILITY_MODE_SYNCHRONOUS", + 2: "DURABILITY_MODE_ASYNCHRONOUS", + } + DurabilityMode_value = map[string]int32{ + "DURABILITY_MODE_UNSPECIFIED": 0, + "DURABILITY_MODE_SYNCHRONOUS": 1, + "DURABILITY_MODE_ASYNCHRONOUS": 2, + } +) + +func (x DurabilityMode) Enum() *DurabilityMode { + p := new(DurabilityMode) + *p = x + return p +} + +func (x DurabilityMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (DurabilityMode) Descriptor() protoreflect.EnumDescriptor { + return file_actor_rpc_proto_enumTypes[0].Descriptor() +} + +func (DurabilityMode) Type() protoreflect.EnumType { + return &file_actor_rpc_proto_enumTypes[0] +} + +func (x DurabilityMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use DurabilityMode.Descriptor instead. +func (DurabilityMode) EnumDescriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{0} +} + +type GetActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetActorRequest) Reset() { + *x = GetActorRequest{} + mi := &file_actor_rpc_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetActorRequest) ProtoMessage() {} + +func (x *GetActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetActorRequest.ProtoReflect.Descriptor instead. +func (*GetActorRequest) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{0} +} + +func (x *GetActorRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +type CreateActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *Actor `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateActorRequest) Reset() { + *x = CreateActorRequest{} + mi := &file_actor_rpc_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateActorRequest) ProtoMessage() {} + +func (x *CreateActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateActorRequest.ProtoReflect.Descriptor instead. +func (*CreateActorRequest) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{1} +} + +func (x *CreateActorRequest) GetActor() *Actor { + if x != nil { + return x.Actor + } + return nil +} + +type SuspendActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + Options *SuspendOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SuspendActorRequest) Reset() { + *x = SuspendActorRequest{} + mi := &file_actor_rpc_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SuspendActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SuspendActorRequest) ProtoMessage() {} + +func (x *SuspendActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SuspendActorRequest.ProtoReflect.Descriptor instead. +func (*SuspendActorRequest) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{2} +} + +func (x *SuspendActorRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +func (x *SuspendActorRequest) GetOptions() *SuspendOptions { + if x != nil { + return x.Options + } + return nil +} + +type SuspendActorResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *Actor `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SuspendActorResponse) Reset() { + *x = SuspendActorResponse{} + mi := &file_actor_rpc_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SuspendActorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SuspendActorResponse) ProtoMessage() {} + +func (x *SuspendActorResponse) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SuspendActorResponse.ProtoReflect.Descriptor instead. +func (*SuspendActorResponse) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{3} +} + +func (x *SuspendActorResponse) GetActor() *Actor { + if x != nil { + return x.Actor + } + return nil +} + +type SuspendOptions struct { + state protoimpl.MessageState `protogen:"open.v1"` + Mode DurabilityMode `protobuf:"varint,1,opt,name=mode,proto3,enum=substrate.ateapi.v1.DurabilityMode" json:"mode,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SuspendOptions) Reset() { + *x = SuspendOptions{} + mi := &file_actor_rpc_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SuspendOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SuspendOptions) ProtoMessage() {} + +func (x *SuspendOptions) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SuspendOptions.ProtoReflect.Descriptor instead. +func (*SuspendOptions) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{4} +} + +func (x *SuspendOptions) GetMode() DurabilityMode { + if x != nil { + return x.Mode + } + return DurabilityMode_DURABILITY_MODE_UNSPECIFIED +} + +type ResumeActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResumeActorRequest) Reset() { + *x = ResumeActorRequest{} + mi := &file_actor_rpc_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResumeActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResumeActorRequest) ProtoMessage() {} + +func (x *ResumeActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResumeActorRequest.ProtoReflect.Descriptor instead. +func (*ResumeActorRequest) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{5} +} + +func (x *ResumeActorRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +type ResumeActorResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *Actor `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + Resumed bool `protobuf:"varint,2,opt,name=resumed,proto3" json:"resumed,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResumeActorResponse) Reset() { + *x = ResumeActorResponse{} + mi := &file_actor_rpc_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResumeActorResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResumeActorResponse) ProtoMessage() {} + +func (x *ResumeActorResponse) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResumeActorResponse.ProtoReflect.Descriptor instead. +func (*ResumeActorResponse) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{6} +} + +func (x *ResumeActorResponse) GetActor() *Actor { + if x != nil { + return x.Actor + } + return nil +} + +func (x *ResumeActorResponse) GetResumed() bool { + if x != nil { + return x.Resumed + } + return false +} + +type DeleteActorRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + Options *DeleteOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteActorRequest) Reset() { + *x = DeleteActorRequest{} + mi := &file_actor_rpc_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteActorRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteActorRequest) ProtoMessage() {} + +func (x *DeleteActorRequest) ProtoReflect() protoreflect.Message { + mi := &file_actor_rpc_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteActorRequest.ProtoReflect.Descriptor instead. +func (*DeleteActorRequest) Descriptor() ([]byte, []int) { + return file_actor_rpc_proto_rawDescGZIP(), []int{7} +} + +func (x *DeleteActorRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +func (x *DeleteActorRequest) GetOptions() *DeleteOptions { + if x != nil { + return x.Options + } + return nil +} + +var File_actor_rpc_proto protoreflect.FileDescriptor + +const file_actor_rpc_proto_rawDesc = "" + + "\n" + + "\x0factor_rpc.proto\x12\x13substrate.ateapi.v1\x1a\vactor.proto\x1a\fcommon.proto\"G\n" + + "\x0fGetActorRequest\x124\n" + + "\x05actor\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x05actor\"F\n" + + "\x12CreateActorRequest\x120\n" + + "\x05actor\x18\x01 \x01(\v2\x1a.substrate.ateapi.v1.ActorR\x05actor\"\x8a\x01\n" + + "\x13SuspendActorRequest\x124\n" + + "\x05actor\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x05actor\x12=\n" + + "\aoptions\x18\x02 \x01(\v2#.substrate.ateapi.v1.SuspendOptionsR\aoptions\"H\n" + + "\x14SuspendActorResponse\x120\n" + + "\x05actor\x18\x01 \x01(\v2\x1a.substrate.ateapi.v1.ActorR\x05actor\"I\n" + + "\x0eSuspendOptions\x127\n" + + "\x04mode\x18\x01 \x01(\x0e2#.substrate.ateapi.v1.DurabilityModeR\x04mode\"J\n" + + "\x12ResumeActorRequest\x124\n" + + "\x05actor\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x05actor\"a\n" + + "\x13ResumeActorResponse\x120\n" + + "\x05actor\x18\x01 \x01(\v2\x1a.substrate.ateapi.v1.ActorR\x05actor\x12\x18\n" + + "\aresumed\x18\x02 \x01(\bR\aresumed\"\x88\x01\n" + + "\x12DeleteActorRequest\x124\n" + + "\x05actor\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x05actor\x12<\n" + + "\aoptions\x18\x02 \x01(\v2\".substrate.ateapi.v1.DeleteOptionsR\aoptions*t\n" + + "\x0eDurabilityMode\x12\x1f\n" + + "\x1bDURABILITY_MODE_UNSPECIFIED\x10\x00\x12\x1f\n" + + "\x1bDURABILITY_MODE_SYNCHRONOUS\x10\x01\x12 \n" + + "\x1cDURABILITY_MODE_ASYNCHRONOUS\x10\x02BDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_actor_rpc_proto_rawDescOnce sync.Once + file_actor_rpc_proto_rawDescData []byte +) + +func file_actor_rpc_proto_rawDescGZIP() []byte { + file_actor_rpc_proto_rawDescOnce.Do(func() { + file_actor_rpc_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_actor_rpc_proto_rawDesc), len(file_actor_rpc_proto_rawDesc))) + }) + return file_actor_rpc_proto_rawDescData +} + +var file_actor_rpc_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_actor_rpc_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_actor_rpc_proto_goTypes = []any{ + (DurabilityMode)(0), // 0: substrate.ateapi.v1.DurabilityMode + (*GetActorRequest)(nil), // 1: substrate.ateapi.v1.GetActorRequest + (*CreateActorRequest)(nil), // 2: substrate.ateapi.v1.CreateActorRequest + (*SuspendActorRequest)(nil), // 3: substrate.ateapi.v1.SuspendActorRequest + (*SuspendActorResponse)(nil), // 4: substrate.ateapi.v1.SuspendActorResponse + (*SuspendOptions)(nil), // 5: substrate.ateapi.v1.SuspendOptions + (*ResumeActorRequest)(nil), // 6: substrate.ateapi.v1.ResumeActorRequest + (*ResumeActorResponse)(nil), // 7: substrate.ateapi.v1.ResumeActorResponse + (*DeleteActorRequest)(nil), // 8: substrate.ateapi.v1.DeleteActorRequest + (*ObjectRef)(nil), // 9: substrate.ateapi.v1.ObjectRef + (*Actor)(nil), // 10: substrate.ateapi.v1.Actor + (*DeleteOptions)(nil), // 11: substrate.ateapi.v1.DeleteOptions +} +var file_actor_rpc_proto_depIdxs = []int32{ + 9, // 0: substrate.ateapi.v1.GetActorRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 10, // 1: substrate.ateapi.v1.CreateActorRequest.actor:type_name -> substrate.ateapi.v1.Actor + 9, // 2: substrate.ateapi.v1.SuspendActorRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 5, // 3: substrate.ateapi.v1.SuspendActorRequest.options:type_name -> substrate.ateapi.v1.SuspendOptions + 10, // 4: substrate.ateapi.v1.SuspendActorResponse.actor:type_name -> substrate.ateapi.v1.Actor + 0, // 5: substrate.ateapi.v1.SuspendOptions.mode:type_name -> substrate.ateapi.v1.DurabilityMode + 9, // 6: substrate.ateapi.v1.ResumeActorRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 10, // 7: substrate.ateapi.v1.ResumeActorResponse.actor:type_name -> substrate.ateapi.v1.Actor + 9, // 8: substrate.ateapi.v1.DeleteActorRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 11, // 9: substrate.ateapi.v1.DeleteActorRequest.options:type_name -> substrate.ateapi.v1.DeleteOptions + 10, // [10:10] is the sub-list for method output_type + 10, // [10:10] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name +} + +func init() { file_actor_rpc_proto_init() } +func file_actor_rpc_proto_init() { + if File_actor_rpc_proto != nil { + return + } + file_actor_proto_init() + file_common_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_actor_rpc_proto_rawDesc), len(file_actor_rpc_proto_rawDesc)), + NumEnums: 1, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_actor_rpc_proto_goTypes, + DependencyIndexes: file_actor_rpc_proto_depIdxs, + EnumInfos: file_actor_rpc_proto_enumTypes, + MessageInfos: file_actor_rpc_proto_msgTypes, + }.Build() + File_actor_rpc_proto = out.File + file_actor_rpc_proto_goTypes = nil + file_actor_rpc_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/actor_rpc.proto b/src/crux/sandbox/substrateapi/actor_rpc.proto new file mode 100644 index 0000000000..1e81978bc1 --- /dev/null +++ b/src/crux/sandbox/substrateapi/actor_rpc.proto @@ -0,0 +1,72 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "actor.proto"; +import "common.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message GetActorRequest { + ObjectRef actor = 1; +} + +message CreateActorRequest { + Actor actor = 1; +} + +message SuspendActorRequest { + ObjectRef actor = 1; + SuspendOptions options = 2; +} + +message SuspendActorResponse { + Actor actor = 1; +} + +message SuspendOptions { + DurabilityMode mode = 1; +} + +enum DurabilityMode { + DURABILITY_MODE_UNSPECIFIED = 0; + DURABILITY_MODE_SYNCHRONOUS = 1; + DURABILITY_MODE_ASYNCHRONOUS = 2; +} + +message ResumeActorRequest { + ObjectRef actor = 1; +} + +message ResumeActorResponse { + Actor actor = 1; + bool resumed = 2; +} + +message DeleteActorRequest { + ObjectRef actor = 1; + DeleteOptions options = 2; +} diff --git a/src/crux/sandbox/substrateapi/buf.gen.yaml b/src/crux/sandbox/substrateapi/buf.gen.yaml new file mode 100644 index 0000000000..c5ca49b889 --- /dev/null +++ b/src/crux/sandbox/substrateapi/buf.gen.yaml @@ -0,0 +1,8 @@ +version: v2 +plugins: + - local: protoc-gen-go + out: . + opt: paths=source_relative + - local: protoc-gen-go-grpc + out: . + opt: paths=source_relative diff --git a/src/crux/sandbox/substrateapi/buf.yaml b/src/crux/sandbox/substrateapi/buf.yaml new file mode 100644 index 0000000000..b8699818a0 --- /dev/null +++ b/src/crux/sandbox/substrateapi/buf.yaml @@ -0,0 +1,3 @@ +version: v2 +modules: + - path: . diff --git a/src/crux/sandbox/substrateapi/common.pb.go b/src/crux/sandbox/substrateapi/common.pb.go new file mode 100644 index 0000000000..02d6dd3697 --- /dev/null +++ b/src/crux/sandbox/substrateapi/common.pb.go @@ -0,0 +1,428 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: common.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + timestamppb "google.golang.org/protobuf/types/known/timestamppb" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Selector struct { + state protoimpl.MessageState `protogen:"open.v1"` + MatchLabels map[string]string `protobuf:"bytes,1,rep,name=match_labels,json=matchLabels,proto3" json:"match_labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Selector) Reset() { + *x = Selector{} + mi := &file_common_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Selector) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Selector) ProtoMessage() {} + +func (x *Selector) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Selector.ProtoReflect.Descriptor instead. +func (*Selector) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{0} +} + +func (x *Selector) GetMatchLabels() map[string]string { + if x != nil { + return x.MatchLabels + } + return nil +} + +type ResourceMetadata struct { + state protoimpl.MessageState `protogen:"open.v1"` + Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + Uid string `protobuf:"bytes,3,opt,name=uid,proto3" json:"uid,omitempty"` + Version int64 `protobuf:"varint,4,opt,name=version,proto3" json:"version,omitempty"` + CreateTime *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=create_time,json=createTime,proto3" json:"create_time,omitempty"` + UpdateTime *timestamppb.Timestamp `protobuf:"bytes,6,opt,name=update_time,json=updateTime,proto3" json:"update_time,omitempty"` + DeleteTime *timestamppb.Timestamp `protobuf:"bytes,7,opt,name=delete_time,json=deleteTime,proto3" json:"delete_time,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResourceMetadata) Reset() { + *x = ResourceMetadata{} + mi := &file_common_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResourceMetadata) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResourceMetadata) ProtoMessage() {} + +func (x *ResourceMetadata) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResourceMetadata.ProtoReflect.Descriptor instead. +func (*ResourceMetadata) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{1} +} + +func (x *ResourceMetadata) GetAtespace() string { + if x != nil { + return x.Atespace + } + return "" +} + +func (x *ResourceMetadata) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *ResourceMetadata) GetUid() string { + if x != nil { + return x.Uid + } + return "" +} + +func (x *ResourceMetadata) GetVersion() int64 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *ResourceMetadata) GetCreateTime() *timestamppb.Timestamp { + if x != nil { + return x.CreateTime + } + return nil +} + +func (x *ResourceMetadata) GetUpdateTime() *timestamppb.Timestamp { + if x != nil { + return x.UpdateTime + } + return nil +} + +func (x *ResourceMetadata) GetDeleteTime() *timestamppb.Timestamp { + if x != nil { + return x.DeleteTime + } + return nil +} + +type ObjectRef struct { + state protoimpl.MessageState `protogen:"open.v1"` + Atespace string `protobuf:"bytes,1,opt,name=atespace,proto3" json:"atespace,omitempty"` + Name string `protobuf:"bytes,2,opt,name=name,proto3" json:"name,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ObjectRef) Reset() { + *x = ObjectRef{} + mi := &file_common_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ObjectRef) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ObjectRef) ProtoMessage() {} + +func (x *ObjectRef) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ObjectRef.ProtoReflect.Descriptor instead. +func (*ObjectRef) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{2} +} + +func (x *ObjectRef) GetAtespace() string { + if x != nil { + return x.Atespace + } + return "" +} + +func (x *ObjectRef) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +type DeleteOptions struct { + state protoimpl.MessageState `protogen:"open.v1"` + Version int64 `protobuf:"varint,1,opt,name=version,proto3" json:"version,omitempty"` + Uid string `protobuf:"bytes,2,opt,name=uid,proto3" json:"uid,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteOptions) Reset() { + *x = DeleteOptions{} + mi := &file_common_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteOptions) ProtoMessage() {} + +func (x *DeleteOptions) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteOptions.ProtoReflect.Descriptor instead. +func (*DeleteOptions) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{3} +} + +func (x *DeleteOptions) GetVersion() int64 { + if x != nil { + return x.Version + } + return 0 +} + +func (x *DeleteOptions) GetUid() string { + if x != nil { + return x.Uid + } + return "" +} + +type ListOptions struct { + state protoimpl.MessageState `protogen:"open.v1"` + PageSize int32 `protobuf:"varint,1,opt,name=page_size,json=pageSize,proto3" json:"page_size,omitempty"` + PageToken string `protobuf:"bytes,2,opt,name=page_token,json=pageToken,proto3" json:"page_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListOptions) Reset() { + *x = ListOptions{} + mi := &file_common_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListOptions) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListOptions) ProtoMessage() {} + +func (x *ListOptions) ProtoReflect() protoreflect.Message { + mi := &file_common_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListOptions.ProtoReflect.Descriptor instead. +func (*ListOptions) Descriptor() ([]byte, []int) { + return file_common_proto_rawDescGZIP(), []int{4} +} + +func (x *ListOptions) GetPageSize() int32 { + if x != nil { + return x.PageSize + } + return 0 +} + +func (x *ListOptions) GetPageToken() string { + if x != nil { + return x.PageToken + } + return "" +} + +var File_common_proto protoreflect.FileDescriptor + +const file_common_proto_rawDesc = "" + + "\n" + + "\fcommon.proto\x12\x13substrate.ateapi.v1\x1a\x1fgoogle/protobuf/timestamp.proto\"\x9d\x01\n" + + "\bSelector\x12Q\n" + + "\fmatch_labels\x18\x01 \x03(\v2..substrate.ateapi.v1.Selector.MatchLabelsEntryR\vmatchLabels\x1a>\n" + + "\x10MatchLabelsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\xa5\x02\n" + + "\x10ResourceMetadata\x12\x1a\n" + + "\batespace\x18\x01 \x01(\tR\batespace\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\x12\x10\n" + + "\x03uid\x18\x03 \x01(\tR\x03uid\x12\x18\n" + + "\aversion\x18\x04 \x01(\x03R\aversion\x12;\n" + + "\vcreate_time\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "createTime\x12;\n" + + "\vupdate_time\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "updateTime\x12;\n" + + "\vdelete_time\x18\a \x01(\v2\x1a.google.protobuf.TimestampR\n" + + "deleteTime\";\n" + + "\tObjectRef\x12\x1a\n" + + "\batespace\x18\x01 \x01(\tR\batespace\x12\x12\n" + + "\x04name\x18\x02 \x01(\tR\x04name\";\n" + + "\rDeleteOptions\x12\x18\n" + + "\aversion\x18\x01 \x01(\x03R\aversion\x12\x10\n" + + "\x03uid\x18\x02 \x01(\tR\x03uid\"I\n" + + "\vListOptions\x12\x1b\n" + + "\tpage_size\x18\x01 \x01(\x05R\bpageSize\x12\x1d\n" + + "\n" + + "page_token\x18\x02 \x01(\tR\tpageTokenBDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_common_proto_rawDescOnce sync.Once + file_common_proto_rawDescData []byte +) + +func file_common_proto_rawDescGZIP() []byte { + file_common_proto_rawDescOnce.Do(func() { + file_common_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_common_proto_rawDesc), len(file_common_proto_rawDesc))) + }) + return file_common_proto_rawDescData +} + +var file_common_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_common_proto_goTypes = []any{ + (*Selector)(nil), // 0: substrate.ateapi.v1.Selector + (*ResourceMetadata)(nil), // 1: substrate.ateapi.v1.ResourceMetadata + (*ObjectRef)(nil), // 2: substrate.ateapi.v1.ObjectRef + (*DeleteOptions)(nil), // 3: substrate.ateapi.v1.DeleteOptions + (*ListOptions)(nil), // 4: substrate.ateapi.v1.ListOptions + nil, // 5: substrate.ateapi.v1.Selector.MatchLabelsEntry + (*timestamppb.Timestamp)(nil), // 6: google.protobuf.Timestamp +} +var file_common_proto_depIdxs = []int32{ + 5, // 0: substrate.ateapi.v1.Selector.match_labels:type_name -> substrate.ateapi.v1.Selector.MatchLabelsEntry + 6, // 1: substrate.ateapi.v1.ResourceMetadata.create_time:type_name -> google.protobuf.Timestamp + 6, // 2: substrate.ateapi.v1.ResourceMetadata.update_time:type_name -> google.protobuf.Timestamp + 6, // 3: substrate.ateapi.v1.ResourceMetadata.delete_time:type_name -> google.protobuf.Timestamp + 4, // [4:4] is the sub-list for method output_type + 4, // [4:4] is the sub-list for method input_type + 4, // [4:4] is the sub-list for extension type_name + 4, // [4:4] is the sub-list for extension extendee + 0, // [0:4] is the sub-list for field type_name +} + +func init() { file_common_proto_init() } +func file_common_proto_init() { + if File_common_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_common_proto_rawDesc), len(file_common_proto_rawDesc)), + NumEnums: 0, + NumMessages: 6, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_common_proto_goTypes, + DependencyIndexes: file_common_proto_depIdxs, + MessageInfos: file_common_proto_msgTypes, + }.Build() + File_common_proto = out.File + file_common_proto_goTypes = nil + file_common_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/common.proto b/src/crux/sandbox/substrateapi/common.proto new file mode 100644 index 0000000000..9ca13d7696 --- /dev/null +++ b/src/crux/sandbox/substrateapi/common.proto @@ -0,0 +1,59 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "google/protobuf/timestamp.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message Selector { + map match_labels = 1; +} + +message ResourceMetadata { + string atespace = 1; + string name = 2; + string uid = 3; + int64 version = 4; + google.protobuf.Timestamp create_time = 5; + google.protobuf.Timestamp update_time = 6; + google.protobuf.Timestamp delete_time = 7; +} + +message ObjectRef { + string atespace = 1; + string name = 2; +} + +message DeleteOptions { + int64 version = 1; + string uid = 2; +} + +message ListOptions { + int32 page_size = 1; + string page_token = 2; +} diff --git a/src/crux/sandbox/substrateapi/container.pb.go b/src/crux/sandbox/substrateapi/container.pb.go new file mode 100644 index 0000000000..7389ce3037 --- /dev/null +++ b/src/crux/sandbox/substrateapi/container.pb.go @@ -0,0 +1,401 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: container.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Container struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Image string `protobuf:"bytes,2,opt,name=image,proto3" json:"image,omitempty"` + Command []string `protobuf:"bytes,3,rep,name=command,proto3" json:"command,omitempty"` + Args []string `protobuf:"bytes,4,rep,name=args,proto3" json:"args,omitempty"` + Env []*EnvVar `protobuf:"bytes,5,rep,name=env,proto3" json:"env,omitempty"` + VolumeMounts []*VolumeMount `protobuf:"bytes,7,rep,name=volume_mounts,json=volumeMounts,proto3" json:"volume_mounts,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Container) Reset() { + *x = Container{} + mi := &file_container_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Container) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Container) ProtoMessage() {} + +func (x *Container) ProtoReflect() protoreflect.Message { + mi := &file_container_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Container.ProtoReflect.Descriptor instead. +func (*Container) Descriptor() ([]byte, []int) { + return file_container_proto_rawDescGZIP(), []int{0} +} + +func (x *Container) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Container) GetImage() string { + if x != nil { + return x.Image + } + return "" +} + +func (x *Container) GetCommand() []string { + if x != nil { + return x.Command + } + return nil +} + +func (x *Container) GetArgs() []string { + if x != nil { + return x.Args + } + return nil +} + +func (x *Container) GetEnv() []*EnvVar { + if x != nil { + return x.Env + } + return nil +} + +func (x *Container) GetVolumeMounts() []*VolumeMount { + if x != nil { + return x.VolumeMounts + } + return nil +} + +type EnvVar struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnvVar) Reset() { + *x = EnvVar{} + mi := &file_container_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnvVar) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnvVar) ProtoMessage() {} + +func (x *EnvVar) ProtoReflect() protoreflect.Message { + mi := &file_container_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EnvVar.ProtoReflect.Descriptor instead. +func (*EnvVar) Descriptor() ([]byte, []int) { + return file_container_proto_rawDescGZIP(), []int{1} +} + +func (x *EnvVar) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *EnvVar) GetValue() string { + if x != nil { + return x.Value + } + return "" +} + +type Volume struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + DurableDir *DurableDirVolumeSource `protobuf:"bytes,2,opt,name=durable_dir,json=durableDir,proto3" json:"durable_dir,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Volume) Reset() { + *x = Volume{} + mi := &file_container_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Volume) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Volume) ProtoMessage() {} + +func (x *Volume) ProtoReflect() protoreflect.Message { + mi := &file_container_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Volume.ProtoReflect.Descriptor instead. +func (*Volume) Descriptor() ([]byte, []int) { + return file_container_proto_rawDescGZIP(), []int{2} +} + +func (x *Volume) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *Volume) GetDurableDir() *DurableDirVolumeSource { + if x != nil { + return x.DurableDir + } + return nil +} + +type DurableDirVolumeSource struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DurableDirVolumeSource) Reset() { + *x = DurableDirVolumeSource{} + mi := &file_container_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DurableDirVolumeSource) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DurableDirVolumeSource) ProtoMessage() {} + +func (x *DurableDirVolumeSource) ProtoReflect() protoreflect.Message { + mi := &file_container_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DurableDirVolumeSource.ProtoReflect.Descriptor instead. +func (*DurableDirVolumeSource) Descriptor() ([]byte, []int) { + return file_container_proto_rawDescGZIP(), []int{3} +} + +type VolumeMount struct { + state protoimpl.MessageState `protogen:"open.v1"` + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + MountPath string `protobuf:"bytes,2,opt,name=mount_path,json=mountPath,proto3" json:"mount_path,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *VolumeMount) Reset() { + *x = VolumeMount{} + mi := &file_container_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *VolumeMount) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*VolumeMount) ProtoMessage() {} + +func (x *VolumeMount) ProtoReflect() protoreflect.Message { + mi := &file_container_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use VolumeMount.ProtoReflect.Descriptor instead. +func (*VolumeMount) Descriptor() ([]byte, []int) { + return file_container_proto_rawDescGZIP(), []int{4} +} + +func (x *VolumeMount) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *VolumeMount) GetMountPath() string { + if x != nil { + return x.MountPath + } + return "" +} + +var File_container_proto protoreflect.FileDescriptor + +const file_container_proto_rawDesc = "" + + "\n" + + "\x0fcontainer.proto\x12\x13substrate.ateapi.v1\"\xd9\x01\n" + + "\tContainer\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x14\n" + + "\x05image\x18\x02 \x01(\tR\x05image\x12\x18\n" + + "\acommand\x18\x03 \x03(\tR\acommand\x12\x12\n" + + "\x04args\x18\x04 \x03(\tR\x04args\x12-\n" + + "\x03env\x18\x05 \x03(\v2\x1b.substrate.ateapi.v1.EnvVarR\x03env\x12E\n" + + "\rvolume_mounts\x18\a \x03(\v2 .substrate.ateapi.v1.VolumeMountR\fvolumeMounts\"7\n" + + "\x06EnvVar\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x19\n" + + "\x05value\x18\x02 \x01(\tB\x03\x80\x01\x01R\x05value\"j\n" + + "\x06Volume\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12L\n" + + "\vdurable_dir\x18\x02 \x01(\v2+.substrate.ateapi.v1.DurableDirVolumeSourceR\n" + + "durableDir\"\x18\n" + + "\x16DurableDirVolumeSource\"@\n" + + "\vVolumeMount\x12\x12\n" + + "\x04name\x18\x01 \x01(\tR\x04name\x12\x1d\n" + + "\n" + + "mount_path\x18\x02 \x01(\tR\tmountPathBDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_container_proto_rawDescOnce sync.Once + file_container_proto_rawDescData []byte +) + +func file_container_proto_rawDescGZIP() []byte { + file_container_proto_rawDescOnce.Do(func() { + file_container_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_container_proto_rawDesc), len(file_container_proto_rawDesc))) + }) + return file_container_proto_rawDescData +} + +var file_container_proto_msgTypes = make([]protoimpl.MessageInfo, 5) +var file_container_proto_goTypes = []any{ + (*Container)(nil), // 0: substrate.ateapi.v1.Container + (*EnvVar)(nil), // 1: substrate.ateapi.v1.EnvVar + (*Volume)(nil), // 2: substrate.ateapi.v1.Volume + (*DurableDirVolumeSource)(nil), // 3: substrate.ateapi.v1.DurableDirVolumeSource + (*VolumeMount)(nil), // 4: substrate.ateapi.v1.VolumeMount +} +var file_container_proto_depIdxs = []int32{ + 1, // 0: substrate.ateapi.v1.Container.env:type_name -> substrate.ateapi.v1.EnvVar + 4, // 1: substrate.ateapi.v1.Container.volume_mounts:type_name -> substrate.ateapi.v1.VolumeMount + 3, // 2: substrate.ateapi.v1.Volume.durable_dir:type_name -> substrate.ateapi.v1.DurableDirVolumeSource + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_container_proto_init() } +func file_container_proto_init() { + if File_container_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_container_proto_rawDesc), len(file_container_proto_rawDesc)), + NumEnums: 0, + NumMessages: 5, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_container_proto_goTypes, + DependencyIndexes: file_container_proto_depIdxs, + MessageInfos: file_container_proto_msgTypes, + }.Build() + File_container_proto = out.File + file_container_proto_goTypes = nil + file_container_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/container.proto b/src/crux/sandbox/substrateapi/container.proto new file mode 100644 index 0000000000..d4d2dbf37f --- /dev/null +++ b/src/crux/sandbox/substrateapi/container.proto @@ -0,0 +1,54 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message Container { + string name = 1; + string image = 2; + repeated string command = 3; + repeated string args = 4; + repeated EnvVar env = 5; + repeated VolumeMount volume_mounts = 7; +} + +message EnvVar { + string name = 1; + string value = 2 [ debug_redact = true ]; +} + +message Volume { + string name = 1; + DurableDirVolumeSource durable_dir = 2; +} + +message DurableDirVolumeSource {} + +message VolumeMount { + string name = 1; + string mount_path = 2; +} diff --git a/src/crux/sandbox/substrateapi/control.pb.go b/src/crux/sandbox/substrateapi/control.pb.go new file mode 100644 index 0000000000..5dab339ffb --- /dev/null +++ b/src/crux/sandbox/substrateapi/control.pb.go @@ -0,0 +1,135 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: control.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +var File_control_proto protoreflect.FileDescriptor + +const file_control_proto_rawDesc = "" + + "\n" + + "\rcontrol.proto\x12\x13substrate.ateapi.v1\x1a\vactor.proto\x1a\x0factor_rpc.proto\x1a\fegress.proto\x1a\x0etemplate.proto\x1a\fworker.proto2\xed\a\n" + + "\aControl\x12N\n" + + "\bGetActor\x12$.substrate.ateapi.v1.GetActorRequest\x1a\x1a.substrate.ateapi.v1.Actor\"\x00\x12T\n" + + "\vCreateActor\x12'.substrate.ateapi.v1.CreateActorRequest\x1a\x1a.substrate.ateapi.v1.Actor\"\x00\x12e\n" + + "\fSuspendActor\x12(.substrate.ateapi.v1.SuspendActorRequest\x1a).substrate.ateapi.v1.SuspendActorResponse\"\x00\x12b\n" + + "\vResumeActor\x12'.substrate.ateapi.v1.ResumeActorRequest\x1a(.substrate.ateapi.v1.ResumeActorResponse\"\x00\x12T\n" + + "\vDeleteActor\x12'.substrate.ateapi.v1.DeleteActorRequest\x1a\x1a.substrate.ateapi.v1.Actor\"\x00\x12s\n" + + "\x17CreateActorEgressPolicy\x123.substrate.ateapi.v1.CreateActorEgressPolicyRequest\x1a!.substrate.ateapi.v1.EgressPolicy\"\x00\x12b\n" + + "\vListWorkers\x12'.substrate.ateapi.v1.ListWorkersRequest\x1a(.substrate.ateapi.v1.ListWorkersResponse\"\x00\x12l\n" + + "\x13CreateActorTemplate\x12/.substrate.ateapi.v1.CreateActorTemplateRequest\x1a\".substrate.ateapi.v1.ActorTemplate\"\x00\x12f\n" + + "\x10GetActorTemplate\x12,.substrate.ateapi.v1.GetActorTemplateRequest\x1a\".substrate.ateapi.v1.ActorTemplate\"\x00\x12l\n" + + "\x13DeleteActorTemplate\x12/.substrate.ateapi.v1.DeleteActorTemplateRequest\x1a\".substrate.ateapi.v1.ActorTemplate\"\x00BDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var file_control_proto_goTypes = []any{ + (*GetActorRequest)(nil), // 0: substrate.ateapi.v1.GetActorRequest + (*CreateActorRequest)(nil), // 1: substrate.ateapi.v1.CreateActorRequest + (*SuspendActorRequest)(nil), // 2: substrate.ateapi.v1.SuspendActorRequest + (*ResumeActorRequest)(nil), // 3: substrate.ateapi.v1.ResumeActorRequest + (*DeleteActorRequest)(nil), // 4: substrate.ateapi.v1.DeleteActorRequest + (*CreateActorEgressPolicyRequest)(nil), // 5: substrate.ateapi.v1.CreateActorEgressPolicyRequest + (*ListWorkersRequest)(nil), // 6: substrate.ateapi.v1.ListWorkersRequest + (*CreateActorTemplateRequest)(nil), // 7: substrate.ateapi.v1.CreateActorTemplateRequest + (*GetActorTemplateRequest)(nil), // 8: substrate.ateapi.v1.GetActorTemplateRequest + (*DeleteActorTemplateRequest)(nil), // 9: substrate.ateapi.v1.DeleteActorTemplateRequest + (*Actor)(nil), // 10: substrate.ateapi.v1.Actor + (*SuspendActorResponse)(nil), // 11: substrate.ateapi.v1.SuspendActorResponse + (*ResumeActorResponse)(nil), // 12: substrate.ateapi.v1.ResumeActorResponse + (*EgressPolicy)(nil), // 13: substrate.ateapi.v1.EgressPolicy + (*ListWorkersResponse)(nil), // 14: substrate.ateapi.v1.ListWorkersResponse + (*ActorTemplate)(nil), // 15: substrate.ateapi.v1.ActorTemplate +} +var file_control_proto_depIdxs = []int32{ + 0, // 0: substrate.ateapi.v1.Control.GetActor:input_type -> substrate.ateapi.v1.GetActorRequest + 1, // 1: substrate.ateapi.v1.Control.CreateActor:input_type -> substrate.ateapi.v1.CreateActorRequest + 2, // 2: substrate.ateapi.v1.Control.SuspendActor:input_type -> substrate.ateapi.v1.SuspendActorRequest + 3, // 3: substrate.ateapi.v1.Control.ResumeActor:input_type -> substrate.ateapi.v1.ResumeActorRequest + 4, // 4: substrate.ateapi.v1.Control.DeleteActor:input_type -> substrate.ateapi.v1.DeleteActorRequest + 5, // 5: substrate.ateapi.v1.Control.CreateActorEgressPolicy:input_type -> substrate.ateapi.v1.CreateActorEgressPolicyRequest + 6, // 6: substrate.ateapi.v1.Control.ListWorkers:input_type -> substrate.ateapi.v1.ListWorkersRequest + 7, // 7: substrate.ateapi.v1.Control.CreateActorTemplate:input_type -> substrate.ateapi.v1.CreateActorTemplateRequest + 8, // 8: substrate.ateapi.v1.Control.GetActorTemplate:input_type -> substrate.ateapi.v1.GetActorTemplateRequest + 9, // 9: substrate.ateapi.v1.Control.DeleteActorTemplate:input_type -> substrate.ateapi.v1.DeleteActorTemplateRequest + 10, // 10: substrate.ateapi.v1.Control.GetActor:output_type -> substrate.ateapi.v1.Actor + 10, // 11: substrate.ateapi.v1.Control.CreateActor:output_type -> substrate.ateapi.v1.Actor + 11, // 12: substrate.ateapi.v1.Control.SuspendActor:output_type -> substrate.ateapi.v1.SuspendActorResponse + 12, // 13: substrate.ateapi.v1.Control.ResumeActor:output_type -> substrate.ateapi.v1.ResumeActorResponse + 10, // 14: substrate.ateapi.v1.Control.DeleteActor:output_type -> substrate.ateapi.v1.Actor + 13, // 15: substrate.ateapi.v1.Control.CreateActorEgressPolicy:output_type -> substrate.ateapi.v1.EgressPolicy + 14, // 16: substrate.ateapi.v1.Control.ListWorkers:output_type -> substrate.ateapi.v1.ListWorkersResponse + 15, // 17: substrate.ateapi.v1.Control.CreateActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate + 15, // 18: substrate.ateapi.v1.Control.GetActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate + 15, // 19: substrate.ateapi.v1.Control.DeleteActorTemplate:output_type -> substrate.ateapi.v1.ActorTemplate + 10, // [10:20] is the sub-list for method output_type + 0, // [0:10] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_control_proto_init() } +func file_control_proto_init() { + if File_control_proto != nil { + return + } + file_actor_proto_init() + file_actor_rpc_proto_init() + file_egress_proto_init() + file_template_proto_init() + file_worker_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_control_proto_rawDesc), len(file_control_proto_rawDesc)), + NumEnums: 0, + NumMessages: 0, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_control_proto_goTypes, + DependencyIndexes: file_control_proto_depIdxs, + }.Build() + File_control_proto = out.File + file_control_proto_goTypes = nil + file_control_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/control.proto b/src/crux/sandbox/substrateapi/control.proto new file mode 100644 index 0000000000..e95d5a6d98 --- /dev/null +++ b/src/crux/sandbox/substrateapi/control.proto @@ -0,0 +1,48 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "actor.proto"; +import "actor_rpc.proto"; +import "egress.proto"; +import "template.proto"; +import "worker.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +// Control is the Agent Substrate control-plane API served by ate-api-server. +service Control { + rpc GetActor(GetActorRequest) returns (Actor) {} + rpc CreateActor(CreateActorRequest) returns (Actor) {} + rpc SuspendActor(SuspendActorRequest) returns (SuspendActorResponse) {} + rpc ResumeActor(ResumeActorRequest) returns (ResumeActorResponse) {} + rpc DeleteActor(DeleteActorRequest) returns (Actor) {} + rpc CreateActorEgressPolicy(CreateActorEgressPolicyRequest) returns (EgressPolicy) {} + rpc ListWorkers(ListWorkersRequest) returns (ListWorkersResponse) {} + rpc CreateActorTemplate(CreateActorTemplateRequest) returns (ActorTemplate) {} + rpc GetActorTemplate(GetActorTemplateRequest) returns (ActorTemplate) {} + rpc DeleteActorTemplate(DeleteActorTemplateRequest) returns (ActorTemplate) {} +} diff --git a/src/crux/sandbox/substrateapi/control_grpc.pb.go b/src/crux/sandbox/substrateapi/control_grpc.pb.go new file mode 100644 index 0000000000..cc4663c9fb --- /dev/null +++ b/src/crux/sandbox/substrateapi/control_grpc.pb.go @@ -0,0 +1,490 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc (unknown) +// source: control.proto + +package substrateapi + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + Control_GetActor_FullMethodName = "/substrate.ateapi.v1.Control/GetActor" + Control_CreateActor_FullMethodName = "/substrate.ateapi.v1.Control/CreateActor" + Control_SuspendActor_FullMethodName = "/substrate.ateapi.v1.Control/SuspendActor" + Control_ResumeActor_FullMethodName = "/substrate.ateapi.v1.Control/ResumeActor" + Control_DeleteActor_FullMethodName = "/substrate.ateapi.v1.Control/DeleteActor" + Control_CreateActorEgressPolicy_FullMethodName = "/substrate.ateapi.v1.Control/CreateActorEgressPolicy" + Control_ListWorkers_FullMethodName = "/substrate.ateapi.v1.Control/ListWorkers" + Control_CreateActorTemplate_FullMethodName = "/substrate.ateapi.v1.Control/CreateActorTemplate" + Control_GetActorTemplate_FullMethodName = "/substrate.ateapi.v1.Control/GetActorTemplate" + Control_DeleteActorTemplate_FullMethodName = "/substrate.ateapi.v1.Control/DeleteActorTemplate" +) + +// ControlClient is the client API for Control service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Control is the Agent Substrate control-plane API served by ate-api-server. +type ControlClient interface { + GetActor(ctx context.Context, in *GetActorRequest, opts ...grpc.CallOption) (*Actor, error) + CreateActor(ctx context.Context, in *CreateActorRequest, opts ...grpc.CallOption) (*Actor, error) + SuspendActor(ctx context.Context, in *SuspendActorRequest, opts ...grpc.CallOption) (*SuspendActorResponse, error) + ResumeActor(ctx context.Context, in *ResumeActorRequest, opts ...grpc.CallOption) (*ResumeActorResponse, error) + DeleteActor(ctx context.Context, in *DeleteActorRequest, opts ...grpc.CallOption) (*Actor, error) + CreateActorEgressPolicy(ctx context.Context, in *CreateActorEgressPolicyRequest, opts ...grpc.CallOption) (*EgressPolicy, error) + ListWorkers(ctx context.Context, in *ListWorkersRequest, opts ...grpc.CallOption) (*ListWorkersResponse, error) + CreateActorTemplate(ctx context.Context, in *CreateActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) + GetActorTemplate(ctx context.Context, in *GetActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) + DeleteActorTemplate(ctx context.Context, in *DeleteActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) +} + +type controlClient struct { + cc grpc.ClientConnInterface +} + +func NewControlClient(cc grpc.ClientConnInterface) ControlClient { + return &controlClient{cc} +} + +func (c *controlClient) GetActor(ctx context.Context, in *GetActorRequest, opts ...grpc.CallOption) (*Actor, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Actor) + err := c.cc.Invoke(ctx, Control_GetActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) CreateActor(ctx context.Context, in *CreateActorRequest, opts ...grpc.CallOption) (*Actor, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Actor) + err := c.cc.Invoke(ctx, Control_CreateActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) SuspendActor(ctx context.Context, in *SuspendActorRequest, opts ...grpc.CallOption) (*SuspendActorResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SuspendActorResponse) + err := c.cc.Invoke(ctx, Control_SuspendActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) ResumeActor(ctx context.Context, in *ResumeActorRequest, opts ...grpc.CallOption) (*ResumeActorResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ResumeActorResponse) + err := c.cc.Invoke(ctx, Control_ResumeActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) DeleteActor(ctx context.Context, in *DeleteActorRequest, opts ...grpc.CallOption) (*Actor, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(Actor) + err := c.cc.Invoke(ctx, Control_DeleteActor_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) CreateActorEgressPolicy(ctx context.Context, in *CreateActorEgressPolicyRequest, opts ...grpc.CallOption) (*EgressPolicy, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(EgressPolicy) + err := c.cc.Invoke(ctx, Control_CreateActorEgressPolicy_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) ListWorkers(ctx context.Context, in *ListWorkersRequest, opts ...grpc.CallOption) (*ListWorkersResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ListWorkersResponse) + err := c.cc.Invoke(ctx, Control_ListWorkers_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) CreateActorTemplate(ctx context.Context, in *CreateActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ActorTemplate) + err := c.cc.Invoke(ctx, Control_CreateActorTemplate_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) GetActorTemplate(ctx context.Context, in *GetActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ActorTemplate) + err := c.cc.Invoke(ctx, Control_GetActorTemplate_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *controlClient) DeleteActorTemplate(ctx context.Context, in *DeleteActorTemplateRequest, opts ...grpc.CallOption) (*ActorTemplate, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(ActorTemplate) + err := c.cc.Invoke(ctx, Control_DeleteActorTemplate_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// ControlServer is the server API for Control service. +// All implementations must embed UnimplementedControlServer +// for forward compatibility. +// +// Control is the Agent Substrate control-plane API served by ate-api-server. +type ControlServer interface { + GetActor(context.Context, *GetActorRequest) (*Actor, error) + CreateActor(context.Context, *CreateActorRequest) (*Actor, error) + SuspendActor(context.Context, *SuspendActorRequest) (*SuspendActorResponse, error) + ResumeActor(context.Context, *ResumeActorRequest) (*ResumeActorResponse, error) + DeleteActor(context.Context, *DeleteActorRequest) (*Actor, error) + CreateActorEgressPolicy(context.Context, *CreateActorEgressPolicyRequest) (*EgressPolicy, error) + ListWorkers(context.Context, *ListWorkersRequest) (*ListWorkersResponse, error) + CreateActorTemplate(context.Context, *CreateActorTemplateRequest) (*ActorTemplate, error) + GetActorTemplate(context.Context, *GetActorTemplateRequest) (*ActorTemplate, error) + DeleteActorTemplate(context.Context, *DeleteActorTemplateRequest) (*ActorTemplate, error) + mustEmbedUnimplementedControlServer() +} + +// UnimplementedControlServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedControlServer struct{} + +func (UnimplementedControlServer) GetActor(context.Context, *GetActorRequest) (*Actor, error) { + return nil, status.Error(codes.Unimplemented, "method GetActor not implemented") +} +func (UnimplementedControlServer) CreateActor(context.Context, *CreateActorRequest) (*Actor, error) { + return nil, status.Error(codes.Unimplemented, "method CreateActor not implemented") +} +func (UnimplementedControlServer) SuspendActor(context.Context, *SuspendActorRequest) (*SuspendActorResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SuspendActor not implemented") +} +func (UnimplementedControlServer) ResumeActor(context.Context, *ResumeActorRequest) (*ResumeActorResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ResumeActor not implemented") +} +func (UnimplementedControlServer) DeleteActor(context.Context, *DeleteActorRequest) (*Actor, error) { + return nil, status.Error(codes.Unimplemented, "method DeleteActor not implemented") +} +func (UnimplementedControlServer) CreateActorEgressPolicy(context.Context, *CreateActorEgressPolicyRequest) (*EgressPolicy, error) { + return nil, status.Error(codes.Unimplemented, "method CreateActorEgressPolicy not implemented") +} +func (UnimplementedControlServer) ListWorkers(context.Context, *ListWorkersRequest) (*ListWorkersResponse, error) { + return nil, status.Error(codes.Unimplemented, "method ListWorkers not implemented") +} +func (UnimplementedControlServer) CreateActorTemplate(context.Context, *CreateActorTemplateRequest) (*ActorTemplate, error) { + return nil, status.Error(codes.Unimplemented, "method CreateActorTemplate not implemented") +} +func (UnimplementedControlServer) GetActorTemplate(context.Context, *GetActorTemplateRequest) (*ActorTemplate, error) { + return nil, status.Error(codes.Unimplemented, "method GetActorTemplate not implemented") +} +func (UnimplementedControlServer) DeleteActorTemplate(context.Context, *DeleteActorTemplateRequest) (*ActorTemplate, error) { + return nil, status.Error(codes.Unimplemented, "method DeleteActorTemplate not implemented") +} +func (UnimplementedControlServer) mustEmbedUnimplementedControlServer() {} +func (UnimplementedControlServer) testEmbeddedByValue() {} + +// UnsafeControlServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to ControlServer will +// result in compilation errors. +type UnsafeControlServer interface { + mustEmbedUnimplementedControlServer() +} + +func RegisterControlServer(s grpc.ServiceRegistrar, srv ControlServer) { + // If the following call panics, it indicates UnimplementedControlServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&Control_ServiceDesc, srv) +} + +func _Control_GetActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).GetActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_GetActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).GetActor(ctx, req.(*GetActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_CreateActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).CreateActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_CreateActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).CreateActor(ctx, req.(*CreateActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_SuspendActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SuspendActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).SuspendActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_SuspendActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).SuspendActor(ctx, req.(*SuspendActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_ResumeActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ResumeActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).ResumeActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_ResumeActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).ResumeActor(ctx, req.(*ResumeActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_DeleteActor_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteActorRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).DeleteActor(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_DeleteActor_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).DeleteActor(ctx, req.(*DeleteActorRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_CreateActorEgressPolicy_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateActorEgressPolicyRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).CreateActorEgressPolicy(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_CreateActorEgressPolicy_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).CreateActorEgressPolicy(ctx, req.(*CreateActorEgressPolicyRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_ListWorkers_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(ListWorkersRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).ListWorkers(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_ListWorkers_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).ListWorkers(ctx, req.(*ListWorkersRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_CreateActorTemplate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(CreateActorTemplateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).CreateActorTemplate(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_CreateActorTemplate_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).CreateActorTemplate(ctx, req.(*CreateActorTemplateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_GetActorTemplate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetActorTemplateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).GetActorTemplate(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_GetActorTemplate_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).GetActorTemplate(ctx, req.(*GetActorTemplateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +func _Control_DeleteActorTemplate_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(DeleteActorTemplateRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(ControlServer).DeleteActorTemplate(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: Control_DeleteActorTemplate_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(ControlServer).DeleteActorTemplate(ctx, req.(*DeleteActorTemplateRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// Control_ServiceDesc is the grpc.ServiceDesc for Control service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var Control_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "substrate.ateapi.v1.Control", + HandlerType: (*ControlServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "GetActor", + Handler: _Control_GetActor_Handler, + }, + { + MethodName: "CreateActor", + Handler: _Control_CreateActor_Handler, + }, + { + MethodName: "SuspendActor", + Handler: _Control_SuspendActor_Handler, + }, + { + MethodName: "ResumeActor", + Handler: _Control_ResumeActor_Handler, + }, + { + MethodName: "DeleteActor", + Handler: _Control_DeleteActor_Handler, + }, + { + MethodName: "CreateActorEgressPolicy", + Handler: _Control_CreateActorEgressPolicy_Handler, + }, + { + MethodName: "ListWorkers", + Handler: _Control_ListWorkers_Handler, + }, + { + MethodName: "CreateActorTemplate", + Handler: _Control_CreateActorTemplate_Handler, + }, + { + MethodName: "GetActorTemplate", + Handler: _Control_GetActorTemplate_Handler, + }, + { + MethodName: "DeleteActorTemplate", + Handler: _Control_DeleteActorTemplate_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "control.proto", +} diff --git a/src/crux/sandbox/substrateapi/egress.pb.go b/src/crux/sandbox/substrateapi/egress.pb.go new file mode 100644 index 0000000000..170d873c8a --- /dev/null +++ b/src/crux/sandbox/substrateapi/egress.pb.go @@ -0,0 +1,553 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: egress.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type EgressPolicy struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metadata *ResourceMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + Rules []*EgressRule `protobuf:"bytes,2,rep,name=rules,proto3" json:"rules,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EgressPolicy) Reset() { + *x = EgressPolicy{} + mi := &file_egress_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EgressPolicy) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EgressPolicy) ProtoMessage() {} + +func (x *EgressPolicy) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EgressPolicy.ProtoReflect.Descriptor instead. +func (*EgressPolicy) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{0} +} + +func (x *EgressPolicy) GetMetadata() *ResourceMetadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *EgressPolicy) GetRules() []*EgressRule { + if x != nil { + return x.Rules + } + return nil +} + +type EgressRule struct { + state protoimpl.MessageState `protogen:"open.v1"` + Http *HttpRule `protobuf:"bytes,1,opt,name=http,proto3" json:"http,omitempty"` + Https *HttpsRule `protobuf:"bytes,2,opt,name=https,proto3" json:"https,omitempty"` + TlsPassthrough *TlsPassthroughRule `protobuf:"bytes,3,opt,name=tls_passthrough,json=tlsPassthrough,proto3" json:"tls_passthrough,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EgressRule) Reset() { + *x = EgressRule{} + mi := &file_egress_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EgressRule) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EgressRule) ProtoMessage() {} + +func (x *EgressRule) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use EgressRule.ProtoReflect.Descriptor instead. +func (*EgressRule) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{1} +} + +func (x *EgressRule) GetHttp() *HttpRule { + if x != nil { + return x.Http + } + return nil +} + +func (x *EgressRule) GetHttps() *HttpsRule { + if x != nil { + return x.Https + } + return nil +} + +func (x *EgressRule) GetTlsPassthrough() *TlsPassthroughRule { + if x != nil { + return x.TlsPassthrough + } + return nil +} + +type HttpRule struct { + state protoimpl.MessageState `protogen:"open.v1"` + Hostnames []string `protobuf:"bytes,1,rep,name=hostnames,proto3" json:"hostnames,omitempty"` + Ports *Ports `protobuf:"bytes,2,opt,name=ports,proto3" json:"ports,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HttpRule) Reset() { + *x = HttpRule{} + mi := &file_egress_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HttpRule) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HttpRule) ProtoMessage() {} + +func (x *HttpRule) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HttpRule.ProtoReflect.Descriptor instead. +func (*HttpRule) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{2} +} + +func (x *HttpRule) GetHostnames() []string { + if x != nil { + return x.Hostnames + } + return nil +} + +func (x *HttpRule) GetPorts() *Ports { + if x != nil { + return x.Ports + } + return nil +} + +type HttpsRule struct { + state protoimpl.MessageState `protogen:"open.v1"` + Hostnames []string `protobuf:"bytes,1,rep,name=hostnames,proto3" json:"hostnames,omitempty"` + Ports *Ports `protobuf:"bytes,2,opt,name=ports,proto3" json:"ports,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *HttpsRule) Reset() { + *x = HttpsRule{} + mi := &file_egress_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *HttpsRule) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*HttpsRule) ProtoMessage() {} + +func (x *HttpsRule) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use HttpsRule.ProtoReflect.Descriptor instead. +func (*HttpsRule) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{3} +} + +func (x *HttpsRule) GetHostnames() []string { + if x != nil { + return x.Hostnames + } + return nil +} + +func (x *HttpsRule) GetPorts() *Ports { + if x != nil { + return x.Ports + } + return nil +} + +type TlsPassthroughRule struct { + state protoimpl.MessageState `protogen:"open.v1"` + Hostnames []string `protobuf:"bytes,1,rep,name=hostnames,proto3" json:"hostnames,omitempty"` + Ports *Ports `protobuf:"bytes,2,opt,name=ports,proto3" json:"ports,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TlsPassthroughRule) Reset() { + *x = TlsPassthroughRule{} + mi := &file_egress_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TlsPassthroughRule) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TlsPassthroughRule) ProtoMessage() {} + +func (x *TlsPassthroughRule) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TlsPassthroughRule.ProtoReflect.Descriptor instead. +func (*TlsPassthroughRule) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{4} +} + +func (x *TlsPassthroughRule) GetHostnames() []string { + if x != nil { + return x.Hostnames + } + return nil +} + +func (x *TlsPassthroughRule) GetPorts() *Ports { + if x != nil { + return x.Ports + } + return nil +} + +type Ports struct { + state protoimpl.MessageState `protogen:"open.v1"` + All *AllPorts `protobuf:"bytes,1,opt,name=all,proto3" json:"all,omitempty"` + Numbers []int32 `protobuf:"varint,2,rep,packed,name=numbers,proto3" json:"numbers,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Ports) Reset() { + *x = Ports{} + mi := &file_egress_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Ports) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Ports) ProtoMessage() {} + +func (x *Ports) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Ports.ProtoReflect.Descriptor instead. +func (*Ports) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{5} +} + +func (x *Ports) GetAll() *AllPorts { + if x != nil { + return x.All + } + return nil +} + +func (x *Ports) GetNumbers() []int32 { + if x != nil { + return x.Numbers + } + return nil +} + +type AllPorts struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *AllPorts) Reset() { + *x = AllPorts{} + mi := &file_egress_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *AllPorts) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*AllPorts) ProtoMessage() {} + +func (x *AllPorts) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use AllPorts.ProtoReflect.Descriptor instead. +func (*AllPorts) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{6} +} + +type CreateActorEgressPolicyRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Actor *ObjectRef `protobuf:"bytes,1,opt,name=actor,proto3" json:"actor,omitempty"` + EgressPolicy *EgressPolicy `protobuf:"bytes,2,opt,name=egress_policy,json=egressPolicy,proto3" json:"egress_policy,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateActorEgressPolicyRequest) Reset() { + *x = CreateActorEgressPolicyRequest{} + mi := &file_egress_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateActorEgressPolicyRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateActorEgressPolicyRequest) ProtoMessage() {} + +func (x *CreateActorEgressPolicyRequest) ProtoReflect() protoreflect.Message { + mi := &file_egress_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateActorEgressPolicyRequest.ProtoReflect.Descriptor instead. +func (*CreateActorEgressPolicyRequest) Descriptor() ([]byte, []int) { + return file_egress_proto_rawDescGZIP(), []int{7} +} + +func (x *CreateActorEgressPolicyRequest) GetActor() *ObjectRef { + if x != nil { + return x.Actor + } + return nil +} + +func (x *CreateActorEgressPolicyRequest) GetEgressPolicy() *EgressPolicy { + if x != nil { + return x.EgressPolicy + } + return nil +} + +var File_egress_proto protoreflect.FileDescriptor + +const file_egress_proto_rawDesc = "" + + "\n" + + "\fegress.proto\x12\x13substrate.ateapi.v1\x1a\fcommon.proto\"\x88\x01\n" + + "\fEgressPolicy\x12A\n" + + "\bmetadata\x18\x01 \x01(\v2%.substrate.ateapi.v1.ResourceMetadataR\bmetadata\x125\n" + + "\x05rules\x18\x02 \x03(\v2\x1f.substrate.ateapi.v1.EgressRuleR\x05rules\"\xc7\x01\n" + + "\n" + + "EgressRule\x121\n" + + "\x04http\x18\x01 \x01(\v2\x1d.substrate.ateapi.v1.HttpRuleR\x04http\x124\n" + + "\x05https\x18\x02 \x01(\v2\x1e.substrate.ateapi.v1.HttpsRuleR\x05https\x12P\n" + + "\x0ftls_passthrough\x18\x03 \x01(\v2'.substrate.ateapi.v1.TlsPassthroughRuleR\x0etlsPassthrough\"Z\n" + + "\bHttpRule\x12\x1c\n" + + "\thostnames\x18\x01 \x03(\tR\thostnames\x120\n" + + "\x05ports\x18\x02 \x01(\v2\x1a.substrate.ateapi.v1.PortsR\x05ports\"[\n" + + "\tHttpsRule\x12\x1c\n" + + "\thostnames\x18\x01 \x03(\tR\thostnames\x120\n" + + "\x05ports\x18\x02 \x01(\v2\x1a.substrate.ateapi.v1.PortsR\x05ports\"d\n" + + "\x12TlsPassthroughRule\x12\x1c\n" + + "\thostnames\x18\x01 \x03(\tR\thostnames\x120\n" + + "\x05ports\x18\x02 \x01(\v2\x1a.substrate.ateapi.v1.PortsR\x05ports\"R\n" + + "\x05Ports\x12/\n" + + "\x03all\x18\x01 \x01(\v2\x1d.substrate.ateapi.v1.AllPortsR\x03all\x12\x18\n" + + "\anumbers\x18\x02 \x03(\x05R\anumbers\"\n" + + "\n" + + "\bAllPorts\"\x9e\x01\n" + + "\x1eCreateActorEgressPolicyRequest\x124\n" + + "\x05actor\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\x05actor\x12F\n" + + "\regress_policy\x18\x02 \x01(\v2!.substrate.ateapi.v1.EgressPolicyR\fegressPolicyBDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_egress_proto_rawDescOnce sync.Once + file_egress_proto_rawDescData []byte +) + +func file_egress_proto_rawDescGZIP() []byte { + file_egress_proto_rawDescOnce.Do(func() { + file_egress_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_egress_proto_rawDesc), len(file_egress_proto_rawDesc))) + }) + return file_egress_proto_rawDescData +} + +var file_egress_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_egress_proto_goTypes = []any{ + (*EgressPolicy)(nil), // 0: substrate.ateapi.v1.EgressPolicy + (*EgressRule)(nil), // 1: substrate.ateapi.v1.EgressRule + (*HttpRule)(nil), // 2: substrate.ateapi.v1.HttpRule + (*HttpsRule)(nil), // 3: substrate.ateapi.v1.HttpsRule + (*TlsPassthroughRule)(nil), // 4: substrate.ateapi.v1.TlsPassthroughRule + (*Ports)(nil), // 5: substrate.ateapi.v1.Ports + (*AllPorts)(nil), // 6: substrate.ateapi.v1.AllPorts + (*CreateActorEgressPolicyRequest)(nil), // 7: substrate.ateapi.v1.CreateActorEgressPolicyRequest + (*ResourceMetadata)(nil), // 8: substrate.ateapi.v1.ResourceMetadata + (*ObjectRef)(nil), // 9: substrate.ateapi.v1.ObjectRef +} +var file_egress_proto_depIdxs = []int32{ + 8, // 0: substrate.ateapi.v1.EgressPolicy.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 1, // 1: substrate.ateapi.v1.EgressPolicy.rules:type_name -> substrate.ateapi.v1.EgressRule + 2, // 2: substrate.ateapi.v1.EgressRule.http:type_name -> substrate.ateapi.v1.HttpRule + 3, // 3: substrate.ateapi.v1.EgressRule.https:type_name -> substrate.ateapi.v1.HttpsRule + 4, // 4: substrate.ateapi.v1.EgressRule.tls_passthrough:type_name -> substrate.ateapi.v1.TlsPassthroughRule + 5, // 5: substrate.ateapi.v1.HttpRule.ports:type_name -> substrate.ateapi.v1.Ports + 5, // 6: substrate.ateapi.v1.HttpsRule.ports:type_name -> substrate.ateapi.v1.Ports + 5, // 7: substrate.ateapi.v1.TlsPassthroughRule.ports:type_name -> substrate.ateapi.v1.Ports + 6, // 8: substrate.ateapi.v1.Ports.all:type_name -> substrate.ateapi.v1.AllPorts + 9, // 9: substrate.ateapi.v1.CreateActorEgressPolicyRequest.actor:type_name -> substrate.ateapi.v1.ObjectRef + 0, // 10: substrate.ateapi.v1.CreateActorEgressPolicyRequest.egress_policy:type_name -> substrate.ateapi.v1.EgressPolicy + 11, // [11:11] is the sub-list for method output_type + 11, // [11:11] is the sub-list for method input_type + 11, // [11:11] is the sub-list for extension type_name + 11, // [11:11] is the sub-list for extension extendee + 0, // [0:11] is the sub-list for field type_name +} + +func init() { file_egress_proto_init() } +func file_egress_proto_init() { + if File_egress_proto != nil { + return + } + file_common_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_egress_proto_rawDesc), len(file_egress_proto_rawDesc)), + NumEnums: 0, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_egress_proto_goTypes, + DependencyIndexes: file_egress_proto_depIdxs, + MessageInfos: file_egress_proto_msgTypes, + }.Build() + File_egress_proto = out.File + file_egress_proto_goTypes = nil + file_egress_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/egress.proto b/src/crux/sandbox/substrateapi/egress.proto new file mode 100644 index 0000000000..8922f27f97 --- /dev/null +++ b/src/crux/sandbox/substrateapi/egress.proto @@ -0,0 +1,68 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "common.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message EgressPolicy { + ResourceMetadata metadata = 1; + repeated EgressRule rules = 2; +} + +message EgressRule { + HttpRule http = 1; + HttpsRule https = 2; + TlsPassthroughRule tls_passthrough = 3; +} + +message HttpRule { + repeated string hostnames = 1; + Ports ports = 2; +} + +message HttpsRule { + repeated string hostnames = 1; + Ports ports = 2; +} + +message TlsPassthroughRule { + repeated string hostnames = 1; + Ports ports = 2; +} + +message Ports { + AllPorts all = 1; + repeated int32 numbers = 2; +} + +message AllPorts {} + +message CreateActorEgressPolicyRequest { + ObjectRef actor = 1; + EgressPolicy egress_policy = 2; +} diff --git a/src/crux/sandbox/substrateapi/generate.sh b/src/crux/sandbox/substrateapi/generate.sh new file mode 100755 index 0000000000..fbfed28814 --- /dev/null +++ b/src/crux/sandbox/substrateapi/generate.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env sh +# Regenerates the *.pb.go files from the trimmed *.proto subset. +# Requires buf, protoc-gen-go (v1.36.x) and protoc-gen-go-grpc on PATH. +set -eu +cd "$(dirname "$0")" +buf generate diff --git a/src/crux/sandbox/substrateapi/substrateapi_test.go b/src/crux/sandbox/substrateapi/substrateapi_test.go new file mode 100644 index 0000000000..68047d96be --- /dev/null +++ b/src/crux/sandbox/substrateapi/substrateapi_test.go @@ -0,0 +1,364 @@ +package substrateapi + +import ( + "context" + "errors" + "net" + "reflect" + "strings" + "testing" + + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/status" + "google.golang.org/grpc/test/bufconn" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" +) + +// The generated code is a wire-compatible subset of Substrate's Control API. +// These tests exercise every message's accessors — empty and populated — and +// every RPC through a real gRPC connection so a regeneration that drops or +// renames a field or method fails here rather than against a cluster. + +// allFiles is a function because the File_* descriptors are populated by +// the generated init functions, after package-level vars. +func allFiles() []protoreflect.FileDescriptor { + return []protoreflect.FileDescriptor{ + File_common_proto, File_actor_proto, File_actor_rpc_proto, File_container_proto, + File_template_proto, File_egress_proto, File_worker_proto, File_control_proto, + } +} + +// callZeroArgMethods invokes every method on v that takes no arguments and +// does not panic on a nil or populated receiver. +func callZeroArgMethods(t *testing.T, v reflect.Value, onlyGetters bool) { + t.Helper() + for i := 0; i < v.NumMethod(); i++ { + m := v.Type().Method(i) + if m.Type.NumIn() != 1 || m.Type.IsVariadic() { + continue + } + if onlyGetters && !strings.HasPrefix(m.Name, "Get") { + continue + } + v.Method(i).Call(nil) + } +} + +// populate sets every field of m to a non-default value, recursing into +// message fields one level so nested getters see data. +func populate(m protoreflect.Message, depth int) { + fields := m.Descriptor().Fields() + for i := 0; i < fields.Len(); i++ { + fd := fields.Get(i) + switch { + case fd.IsList(): + list := m.Mutable(fd).List() + list.Append(sampleValue(fd, list.NewElement, depth)) + case fd.IsMap(): + mp := m.Mutable(fd).Map() + mp.Set(protoreflect.ValueOfString("k").MapKey(), sampleValue(fd.MapValue(), mp.NewValue, depth)) + default: + m.Set(fd, sampleValue(fd, func() protoreflect.Value { return m.NewField(fd) }, depth)) + } + } +} + +func sampleValue(fd protoreflect.FieldDescriptor, newVal func() protoreflect.Value, depth int) protoreflect.Value { + switch fd.Kind() { + case protoreflect.BoolKind: + return protoreflect.ValueOfBool(true) + case protoreflect.StringKind: + return protoreflect.ValueOfString("s") + case protoreflect.BytesKind: + return protoreflect.ValueOfBytes([]byte("b")) + case protoreflect.Int32Kind, protoreflect.Sint32Kind, protoreflect.Sfixed32Kind: + return protoreflect.ValueOfInt32(7) + case protoreflect.Int64Kind, protoreflect.Sint64Kind, protoreflect.Sfixed64Kind: + return protoreflect.ValueOfInt64(7) + case protoreflect.Uint32Kind, protoreflect.Fixed32Kind: + return protoreflect.ValueOfUint32(7) + case protoreflect.Uint64Kind, protoreflect.Fixed64Kind: + return protoreflect.ValueOfUint64(7) + case protoreflect.FloatKind: + return protoreflect.ValueOfFloat32(1.5) + case protoreflect.DoubleKind: + return protoreflect.ValueOfFloat64(1.5) + case protoreflect.EnumKind: + vals := fd.Enum().Values() + return protoreflect.ValueOfEnum(vals.Get(vals.Len() - 1).Number()) + case protoreflect.MessageKind, protoreflect.GroupKind: + v := newVal() + if depth > 0 { + populate(v.Message(), depth-1) + } + return v + } + return newVal() +} + +func TestEveryMessageAccessorAndRoundTrip(t *testing.T) { + seen := 0 + for _, file := range allFiles() { + msgs := file.Messages() + for i := 0; i < msgs.Len(); i++ { + md := msgs.Get(i) + mt, err := protoregistry.GlobalTypes.FindMessageByName(md.FullName()) + if err != nil { + t.Fatalf("%s not registered: %v", md.FullName(), err) + } + seen++ + // Nil receiver: every getter must return the zero value. + nilMsg := reflect.New(reflect.TypeOf(mt.Zero().Interface()).Elem()) + callZeroArgMethods(t, reflect.Zero(nilMsg.Type()), true) + + // Populated receiver, then a marshal/unmarshal round trip. + full := mt.New() + populate(full, 2) + goVal := reflect.ValueOf(full.Interface()) + callZeroArgMethods(t, goVal, false) + raw, err := proto.Marshal(full.Interface()) + if err != nil { + t.Fatalf("%s marshal: %v", md.FullName(), err) + } + back := mt.New().Interface() + if err := proto.Unmarshal(raw, back); err != nil { + t.Fatalf("%s unmarshal: %v", md.FullName(), err) + } + if !proto.Equal(full.Interface(), back) { + t.Errorf("%s round trip lost data", md.FullName()) + } + // Reset then String on the emptied message. + proto.Reset(back) + callZeroArgMethods(t, reflect.ValueOf(back), false) + } + enums := file.Enums() + for i := 0; i < enums.Len(); i++ { + ed := enums.Get(i) + et, err := protoregistry.GlobalTypes.FindEnumByName(ed.FullName()) + if err != nil { + t.Fatalf("%s not registered: %v", ed.FullName(), err) + } + for j := 0; j < ed.Values().Len(); j++ { + e := et.New(ed.Values().Get(j).Number()) + v := reflect.ValueOf(e) + callZeroArgMethods(t, v, false) + p := reflect.New(v.Type()) + p.Elem().Set(v) + callZeroArgMethods(t, p, false) + if s := v.MethodByName("String").Call(nil)[0].String(); s == "" { + t.Errorf("%s value %d has empty String()", ed.FullName(), j) + } + } + } + } + if seen < 40 { + t.Errorf("only %d messages found; the subset shrank unexpectedly", seen) + } +} + +func TestFieldNumbersMatchUpstream(t *testing.T) { + // A few load-bearing field numbers from Substrate's ateapi/v1 protos; + // renumbering any of these would silently talk past the server. + checks := map[string]map[string]int32{ + "substrate.ateapi.v1.ObjectRef": {"atespace": 1, "name": 2}, + "substrate.ateapi.v1.ResourceMetadata": {"atespace": 1, "name": 2}, + "substrate.ateapi.v1.Actor": {"metadata": 1, "actor_template": 2, "worker_selector": 3, "status": 5}, + "substrate.ateapi.v1.Container": {"name": 1, "image": 2, "command": 3, "env": 5, "volume_mounts": 7}, + } + for name, fields := range checks { + md, err := protoregistry.GlobalFiles.FindDescriptorByName(protoreflect.FullName(name)) + if err != nil { + t.Fatalf("%s: %v", name, err) + } + m := md.(protoreflect.MessageDescriptor) + for fname, want := range fields { + fd := m.Fields().ByName(protoreflect.Name(fname)) + if fd == nil { + t.Errorf("%s.%s missing", name, fname) + continue + } + if int32(fd.Number()) != want { + t.Errorf("%s.%s = %d, want %d", name, fname, fd.Number(), want) + } + } + } +} + +// fakeServer answers every RPC by echoing the request's target back. +type fakeServer struct { + UnimplementedControlServer + calls []string +} + +func (f *fakeServer) note(name string) { f.calls = append(f.calls, name) } + +func (f *fakeServer) GetActor(_ context.Context, r *GetActorRequest) (*Actor, error) { + f.note("GetActor") + return &Actor{Metadata: &ResourceMetadata{Name: r.GetActor().GetName()}, Status: &ActorStatus{State: ActorState_ACTOR_STATE_RUNNING}}, nil +} +func (f *fakeServer) CreateActor(_ context.Context, r *CreateActorRequest) (*Actor, error) { + f.note("CreateActor") + return r.GetActor(), nil +} +func (f *fakeServer) SuspendActor(_ context.Context, r *SuspendActorRequest) (*SuspendActorResponse, error) { + f.note("SuspendActor:" + r.GetOptions().GetMode().String()) + return &SuspendActorResponse{}, nil +} +func (f *fakeServer) ResumeActor(context.Context, *ResumeActorRequest) (*ResumeActorResponse, error) { + f.note("ResumeActor") + return &ResumeActorResponse{}, nil +} +func (f *fakeServer) DeleteActor(_ context.Context, r *DeleteActorRequest) (*Actor, error) { + f.note("DeleteActor") + return &Actor{Metadata: &ResourceMetadata{Name: r.GetActor().GetName()}}, nil +} +func (f *fakeServer) CreateActorEgressPolicy(_ context.Context, r *CreateActorEgressPolicyRequest) (*EgressPolicy, error) { + f.note("CreateActorEgressPolicy") + return r.GetEgressPolicy(), nil +} +func (f *fakeServer) ListWorkers(_ context.Context, r *ListWorkersRequest) (*ListWorkersResponse, error) { + f.note("ListWorkers:" + r.GetOptions().GetPageToken()) + if r.GetOptions().GetPageToken() == "" { + return &ListWorkersResponse{Workers: []*Worker{{Metadata: &ResourceMetadata{Name: "w1"}}}, NextPageToken: "p2"}, nil + } + return &ListWorkersResponse{Workers: []*Worker{{Metadata: &ResourceMetadata{Name: "w2"}}}}, nil +} +func (f *fakeServer) CreateActorTemplate(_ context.Context, r *CreateActorTemplateRequest) (*ActorTemplate, error) { + f.note("CreateActorTemplate") + return r.GetActorTemplate(), nil +} +func (f *fakeServer) GetActorTemplate(_ context.Context, r *GetActorTemplateRequest) (*ActorTemplate, error) { + f.note("GetActorTemplate") + return &ActorTemplate{Metadata: &ResourceMetadata{Name: r.GetActorTemplate().GetName()}}, nil +} +func (f *fakeServer) DeleteActorTemplate(_ context.Context, r *DeleteActorTemplateRequest) (*ActorTemplate, error) { + f.note("DeleteActorTemplate") + return &ActorTemplate{Metadata: &ResourceMetadata{Name: r.GetActorTemplate().GetName()}}, nil +} + +func dialBufconn(t *testing.T, srv ControlServer) ControlClient { + t.Helper() + ln := bufconn.Listen(1 << 20) + gs := grpc.NewServer() + RegisterControlServer(gs, srv) + go func() { _ = gs.Serve(ln) }() + t.Cleanup(gs.Stop) + conn, err := grpc.NewClient("passthrough:///bufconn", + grpc.WithContextDialer(func(ctx context.Context, _ string) (net.Conn, error) { return ln.DialContext(ctx) }), + grpc.WithTransportCredentials(insecure.NewCredentials())) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = conn.Close() }) + return NewControlClient(conn) +} + +func TestControlClientRoundTripsEveryRPC(t *testing.T) { + srv := &fakeServer{} + c := dialBufconn(t, srv) + ctx := context.Background() + ref := &ObjectRef{Atespace: "hive", Name: "a1"} + + tpl, err := c.CreateActorTemplate(ctx, &CreateActorTemplateRequest{ActorTemplate: &ActorTemplate{Metadata: &ResourceMetadata{Name: "t1"}}}) + if err != nil || tpl.GetMetadata().GetName() != "t1" { + t.Fatalf("CreateActorTemplate: %v %v", tpl, err) + } + if got, err := c.GetActorTemplate(ctx, &GetActorTemplateRequest{ActorTemplate: ref}); err != nil || got.GetMetadata().GetName() != "a1" { + t.Fatalf("GetActorTemplate: %v %v", got, err) + } + if got, err := c.DeleteActorTemplate(ctx, &DeleteActorTemplateRequest{ActorTemplate: ref}); err != nil || got.GetMetadata().GetName() != "a1" { + t.Fatalf("DeleteActorTemplate: %v %v", got, err) + } + actor, err := c.CreateActor(ctx, &CreateActorRequest{Actor: &Actor{Metadata: &ResourceMetadata{Name: "a1"}, ActorTemplate: ref}}) + if err != nil || actor.GetActorTemplate().GetName() != "a1" { + t.Fatalf("CreateActor: %v %v", actor, err) + } + if got, err := c.GetActor(ctx, &GetActorRequest{Actor: ref}); err != nil || got.GetStatus().GetState() != ActorState_ACTOR_STATE_RUNNING { + t.Fatalf("GetActor: %v %v", got, err) + } + if _, err := c.ResumeActor(ctx, &ResumeActorRequest{Actor: ref}); err != nil { + t.Fatalf("ResumeActor: %v", err) + } + if _, err := c.SuspendActor(ctx, &SuspendActorRequest{Actor: ref, Options: &SuspendOptions{Mode: DurabilityMode_DURABILITY_MODE_ASYNCHRONOUS}}); err != nil { + t.Fatalf("SuspendActor: %v", err) + } + pol, err := c.CreateActorEgressPolicy(ctx, &CreateActorEgressPolicyRequest{Actor: ref, EgressPolicy: &EgressPolicy{Metadata: &ResourceMetadata{Name: "default"}}}) + if err != nil || pol.GetMetadata().GetName() != "default" { + t.Fatalf("CreateActorEgressPolicy: %v %v", pol, err) + } + if got, err := c.DeleteActor(ctx, &DeleteActorRequest{Actor: ref}); err != nil || got.GetMetadata().GetName() != "a1" { + t.Fatalf("DeleteActor: %v %v", got, err) + } + page1, err := c.ListWorkers(ctx, &ListWorkersRequest{Options: &ListOptions{}}) + if err != nil || page1.GetNextPageToken() != "p2" || len(page1.GetWorkers()) != 1 { + t.Fatalf("ListWorkers page 1: %v %v", page1, err) + } + page2, err := c.ListWorkers(ctx, &ListWorkersRequest{Options: &ListOptions{PageToken: "p2"}}) + if err != nil || page2.GetNextPageToken() != "" || page2.GetWorkers()[0].GetMetadata().GetName() != "w2" { + t.Fatalf("ListWorkers page 2: %v %v", page2, err) + } + joined := strings.Join(srv.calls, ",") + for _, want := range []string{"CreateActorTemplate", "GetActorTemplate", "DeleteActorTemplate", "CreateActor", "GetActor", "ResumeActor", "SuspendActor:DURABILITY_MODE_ASYNCHRONOUS", "CreateActorEgressPolicy", "DeleteActor", "ListWorkers:", "ListWorkers:p2"} { + if !strings.Contains(joined, want) { + t.Errorf("server never saw %s: %s", want, joined) + } + } +} + +func TestUnimplementedServerAnswersUnimplemented(t *testing.T) { + c := dialBufconn(t, UnimplementedControlServer{}) + ctx := context.Background() + calls := []func() error{ + func() error { _, err := c.GetActor(ctx, &GetActorRequest{}); return err }, + func() error { _, err := c.CreateActor(ctx, &CreateActorRequest{}); return err }, + func() error { _, err := c.SuspendActor(ctx, &SuspendActorRequest{}); return err }, + func() error { _, err := c.ResumeActor(ctx, &ResumeActorRequest{}); return err }, + func() error { _, err := c.DeleteActor(ctx, &DeleteActorRequest{}); return err }, + func() error { _, err := c.CreateActorEgressPolicy(ctx, &CreateActorEgressPolicyRequest{}); return err }, + func() error { _, err := c.ListWorkers(ctx, &ListWorkersRequest{}); return err }, + func() error { _, err := c.CreateActorTemplate(ctx, &CreateActorTemplateRequest{}); return err }, + func() error { _, err := c.GetActorTemplate(ctx, &GetActorTemplateRequest{}); return err }, + func() error { _, err := c.DeleteActorTemplate(ctx, &DeleteActorTemplateRequest{}); return err }, + } + for i, call := range calls { + err := call() + if st, ok := status.FromError(err); !ok || st.Code() != codes.Unimplemented { + t.Errorf("call %d: %v, want Unimplemented", i, err) + } + } + var u UnimplementedControlServer + u.testEmbeddedByValue() + if _, err := u.GetActor(ctx, nil); err == nil { + t.Error("direct GetActor on the unimplemented server returned nil error") + } + var _ UnsafeControlServer = &fakeServer{} +} + +func TestRegisterControlServerRejectsNilEmbedding(t *testing.T) { + // Registering a server whose embedded UnimplementedControlServer is a + // nil pointer must panic, which is grpc-go's guard against a forgotten + // embedding. + defer func() { + if r := recover(); r == nil { + t.Error("no panic for a nil embedded UnimplementedControlServer") + } + }() + type nilEmbed struct{ *UnimplementedControlServer } + RegisterControlServer(grpc.NewServer(), nilEmbed{}) +} + +func TestServiceDescriptor(t *testing.T) { + if Control_ServiceDesc.ServiceName != "substrate.ateapi.v1.Control" { + t.Errorf("service name = %q", Control_ServiceDesc.ServiceName) + } + if len(Control_ServiceDesc.Methods) != 10 { + t.Errorf("methods = %d, want 10", len(Control_ServiceDesc.Methods)) + } + if !errors.Is(context.Canceled, context.Canceled) { + t.Fatal("unreachable") + } +} diff --git a/src/crux/sandbox/substrateapi/template.pb.go b/src/crux/sandbox/substrateapi/template.pb.go new file mode 100644 index 0000000000..b27724d605 --- /dev/null +++ b/src/crux/sandbox/substrateapi/template.pb.go @@ -0,0 +1,616 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: template.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type GoldenSnapshotMode int32 + +const ( + GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_UNSPECIFIED GoldenSnapshotMode = 0 + GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_ENABLED GoldenSnapshotMode = 1 + GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_DISABLED GoldenSnapshotMode = 2 +) + +// Enum value maps for GoldenSnapshotMode. +var ( + GoldenSnapshotMode_name = map[int32]string{ + 0: "GOLDEN_SNAPSHOT_MODE_UNSPECIFIED", + 1: "GOLDEN_SNAPSHOT_MODE_ENABLED", + 2: "GOLDEN_SNAPSHOT_MODE_DISABLED", + } + GoldenSnapshotMode_value = map[string]int32{ + "GOLDEN_SNAPSHOT_MODE_UNSPECIFIED": 0, + "GOLDEN_SNAPSHOT_MODE_ENABLED": 1, + "GOLDEN_SNAPSHOT_MODE_DISABLED": 2, + } +) + +func (x GoldenSnapshotMode) Enum() *GoldenSnapshotMode { + p := new(GoldenSnapshotMode) + *p = x + return p +} + +func (x GoldenSnapshotMode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (GoldenSnapshotMode) Descriptor() protoreflect.EnumDescriptor { + return file_template_proto_enumTypes[0].Descriptor() +} + +func (GoldenSnapshotMode) Type() protoreflect.EnumType { + return &file_template_proto_enumTypes[0] +} + +func (x GoldenSnapshotMode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use GoldenSnapshotMode.Descriptor instead. +func (GoldenSnapshotMode) EnumDescriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{0} +} + +type ActorTemplate struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metadata *ResourceMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + WorkerSelector *Selector `protobuf:"bytes,2,opt,name=worker_selector,json=workerSelector,proto3" json:"worker_selector,omitempty"` + Containers []*Container `protobuf:"bytes,3,rep,name=containers,proto3" json:"containers,omitempty"` + Volumes []*Volume `protobuf:"bytes,4,rep,name=volumes,proto3" json:"volumes,omitempty"` + SnapshotConfig *SnapshotConfig `protobuf:"bytes,5,opt,name=snapshot_config,json=snapshotConfig,proto3" json:"snapshot_config,omitempty"` + Resources *ResourceRequirements `protobuf:"bytes,6,opt,name=resources,proto3" json:"resources,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ActorTemplate) Reset() { + *x = ActorTemplate{} + mi := &file_template_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ActorTemplate) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ActorTemplate) ProtoMessage() {} + +func (x *ActorTemplate) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ActorTemplate.ProtoReflect.Descriptor instead. +func (*ActorTemplate) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{0} +} + +func (x *ActorTemplate) GetMetadata() *ResourceMetadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *ActorTemplate) GetWorkerSelector() *Selector { + if x != nil { + return x.WorkerSelector + } + return nil +} + +func (x *ActorTemplate) GetContainers() []*Container { + if x != nil { + return x.Containers + } + return nil +} + +func (x *ActorTemplate) GetVolumes() []*Volume { + if x != nil { + return x.Volumes + } + return nil +} + +func (x *ActorTemplate) GetSnapshotConfig() *SnapshotConfig { + if x != nil { + return x.SnapshotConfig + } + return nil +} + +func (x *ActorTemplate) GetResources() *ResourceRequirements { + if x != nil { + return x.Resources + } + return nil +} + +type ResourceRequirements struct { + state protoimpl.MessageState `protogen:"open.v1"` + Limits *Resources `protobuf:"bytes,1,opt,name=limits,proto3" json:"limits,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ResourceRequirements) Reset() { + *x = ResourceRequirements{} + mi := &file_template_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ResourceRequirements) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ResourceRequirements) ProtoMessage() {} + +func (x *ResourceRequirements) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ResourceRequirements.ProtoReflect.Descriptor instead. +func (*ResourceRequirements) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{1} +} + +func (x *ResourceRequirements) GetLimits() *Resources { + if x != nil { + return x.Limits + } + return nil +} + +type Resources struct { + state protoimpl.MessageState `protogen:"open.v1"` + CpuMillicores int64 `protobuf:"varint,1,opt,name=cpu_millicores,json=cpuMillicores,proto3" json:"cpu_millicores,omitempty"` + MemoryBytes int64 `protobuf:"varint,2,opt,name=memory_bytes,json=memoryBytes,proto3" json:"memory_bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Resources) Reset() { + *x = Resources{} + mi := &file_template_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Resources) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Resources) ProtoMessage() {} + +func (x *Resources) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Resources.ProtoReflect.Descriptor instead. +func (*Resources) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{2} +} + +func (x *Resources) GetCpuMillicores() int64 { + if x != nil { + return x.CpuMillicores + } + return 0 +} + +func (x *Resources) GetMemoryBytes() int64 { + if x != nil { + return x.MemoryBytes + } + return 0 +} + +type SnapshotConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + GoldenSnapshotConfig *GoldenSnapshotConfig `protobuf:"bytes,3,opt,name=golden_snapshot_config,json=goldenSnapshotConfig,proto3" json:"golden_snapshot_config,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SnapshotConfig) Reset() { + *x = SnapshotConfig{} + mi := &file_template_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SnapshotConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SnapshotConfig) ProtoMessage() {} + +func (x *SnapshotConfig) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SnapshotConfig.ProtoReflect.Descriptor instead. +func (*SnapshotConfig) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{3} +} + +func (x *SnapshotConfig) GetGoldenSnapshotConfig() *GoldenSnapshotConfig { + if x != nil { + return x.GoldenSnapshotConfig + } + return nil +} + +type GoldenSnapshotConfig struct { + state protoimpl.MessageState `protogen:"open.v1"` + Mode GoldenSnapshotMode `protobuf:"varint,1,opt,name=mode,proto3,enum=substrate.ateapi.v1.GoldenSnapshotMode" json:"mode,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GoldenSnapshotConfig) Reset() { + *x = GoldenSnapshotConfig{} + mi := &file_template_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GoldenSnapshotConfig) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GoldenSnapshotConfig) ProtoMessage() {} + +func (x *GoldenSnapshotConfig) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GoldenSnapshotConfig.ProtoReflect.Descriptor instead. +func (*GoldenSnapshotConfig) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{4} +} + +func (x *GoldenSnapshotConfig) GetMode() GoldenSnapshotMode { + if x != nil { + return x.Mode + } + return GoldenSnapshotMode_GOLDEN_SNAPSHOT_MODE_UNSPECIFIED +} + +type CreateActorTemplateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ActorTemplate *ActorTemplate `protobuf:"bytes,1,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *CreateActorTemplateRequest) Reset() { + *x = CreateActorTemplateRequest{} + mi := &file_template_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *CreateActorTemplateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*CreateActorTemplateRequest) ProtoMessage() {} + +func (x *CreateActorTemplateRequest) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use CreateActorTemplateRequest.ProtoReflect.Descriptor instead. +func (*CreateActorTemplateRequest) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{5} +} + +func (x *CreateActorTemplateRequest) GetActorTemplate() *ActorTemplate { + if x != nil { + return x.ActorTemplate + } + return nil +} + +type GetActorTemplateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ActorTemplate *ObjectRef `protobuf:"bytes,1,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *GetActorTemplateRequest) Reset() { + *x = GetActorTemplateRequest{} + mi := &file_template_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *GetActorTemplateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetActorTemplateRequest) ProtoMessage() {} + +func (x *GetActorTemplateRequest) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetActorTemplateRequest.ProtoReflect.Descriptor instead. +func (*GetActorTemplateRequest) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{6} +} + +func (x *GetActorTemplateRequest) GetActorTemplate() *ObjectRef { + if x != nil { + return x.ActorTemplate + } + return nil +} + +type DeleteActorTemplateRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + ActorTemplate *ObjectRef `protobuf:"bytes,1,opt,name=actor_template,json=actorTemplate,proto3" json:"actor_template,omitempty"` + Options *DeleteOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DeleteActorTemplateRequest) Reset() { + *x = DeleteActorTemplateRequest{} + mi := &file_template_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DeleteActorTemplateRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DeleteActorTemplateRequest) ProtoMessage() {} + +func (x *DeleteActorTemplateRequest) ProtoReflect() protoreflect.Message { + mi := &file_template_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DeleteActorTemplateRequest.ProtoReflect.Descriptor instead. +func (*DeleteActorTemplateRequest) Descriptor() ([]byte, []int) { + return file_template_proto_rawDescGZIP(), []int{7} +} + +func (x *DeleteActorTemplateRequest) GetActorTemplate() *ObjectRef { + if x != nil { + return x.ActorTemplate + } + return nil +} + +func (x *DeleteActorTemplateRequest) GetOptions() *DeleteOptions { + if x != nil { + return x.Options + } + return nil +} + +var File_template_proto protoreflect.FileDescriptor + +const file_template_proto_rawDesc = "" + + "\n" + + "\x0etemplate.proto\x12\x13substrate.ateapi.v1\x1a\fcommon.proto\x1a\x0fcontainer.proto\"\xa8\x03\n" + + "\rActorTemplate\x12A\n" + + "\bmetadata\x18\x01 \x01(\v2%.substrate.ateapi.v1.ResourceMetadataR\bmetadata\x12F\n" + + "\x0fworker_selector\x18\x02 \x01(\v2\x1d.substrate.ateapi.v1.SelectorR\x0eworkerSelector\x12>\n" + + "\n" + + "containers\x18\x03 \x03(\v2\x1e.substrate.ateapi.v1.ContainerR\n" + + "containers\x125\n" + + "\avolumes\x18\x04 \x03(\v2\x1b.substrate.ateapi.v1.VolumeR\avolumes\x12L\n" + + "\x0fsnapshot_config\x18\x05 \x01(\v2#.substrate.ateapi.v1.SnapshotConfigR\x0esnapshotConfig\x12G\n" + + "\tresources\x18\x06 \x01(\v2).substrate.ateapi.v1.ResourceRequirementsR\tresources\"N\n" + + "\x14ResourceRequirements\x126\n" + + "\x06limits\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ResourcesR\x06limits\"U\n" + + "\tResources\x12%\n" + + "\x0ecpu_millicores\x18\x01 \x01(\x03R\rcpuMillicores\x12!\n" + + "\fmemory_bytes\x18\x02 \x01(\x03R\vmemoryBytes\"q\n" + + "\x0eSnapshotConfig\x12_\n" + + "\x16golden_snapshot_config\x18\x03 \x01(\v2).substrate.ateapi.v1.GoldenSnapshotConfigR\x14goldenSnapshotConfig\"S\n" + + "\x14GoldenSnapshotConfig\x12;\n" + + "\x04mode\x18\x01 \x01(\x0e2'.substrate.ateapi.v1.GoldenSnapshotModeR\x04mode\"g\n" + + "\x1aCreateActorTemplateRequest\x12I\n" + + "\x0eactor_template\x18\x01 \x01(\v2\".substrate.ateapi.v1.ActorTemplateR\ractorTemplate\"`\n" + + "\x17GetActorTemplateRequest\x12E\n" + + "\x0eactor_template\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\ractorTemplate\"\xa1\x01\n" + + "\x1aDeleteActorTemplateRequest\x12E\n" + + "\x0eactor_template\x18\x01 \x01(\v2\x1e.substrate.ateapi.v1.ObjectRefR\ractorTemplate\x12<\n" + + "\aoptions\x18\x02 \x01(\v2\".substrate.ateapi.v1.DeleteOptionsR\aoptions*\x7f\n" + + "\x12GoldenSnapshotMode\x12$\n" + + " GOLDEN_SNAPSHOT_MODE_UNSPECIFIED\x10\x00\x12 \n" + + "\x1cGOLDEN_SNAPSHOT_MODE_ENABLED\x10\x01\x12!\n" + + "\x1dGOLDEN_SNAPSHOT_MODE_DISABLED\x10\x02BDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_template_proto_rawDescOnce sync.Once + file_template_proto_rawDescData []byte +) + +func file_template_proto_rawDescGZIP() []byte { + file_template_proto_rawDescOnce.Do(func() { + file_template_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_template_proto_rawDesc), len(file_template_proto_rawDesc))) + }) + return file_template_proto_rawDescData +} + +var file_template_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_template_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_template_proto_goTypes = []any{ + (GoldenSnapshotMode)(0), // 0: substrate.ateapi.v1.GoldenSnapshotMode + (*ActorTemplate)(nil), // 1: substrate.ateapi.v1.ActorTemplate + (*ResourceRequirements)(nil), // 2: substrate.ateapi.v1.ResourceRequirements + (*Resources)(nil), // 3: substrate.ateapi.v1.Resources + (*SnapshotConfig)(nil), // 4: substrate.ateapi.v1.SnapshotConfig + (*GoldenSnapshotConfig)(nil), // 5: substrate.ateapi.v1.GoldenSnapshotConfig + (*CreateActorTemplateRequest)(nil), // 6: substrate.ateapi.v1.CreateActorTemplateRequest + (*GetActorTemplateRequest)(nil), // 7: substrate.ateapi.v1.GetActorTemplateRequest + (*DeleteActorTemplateRequest)(nil), // 8: substrate.ateapi.v1.DeleteActorTemplateRequest + (*ResourceMetadata)(nil), // 9: substrate.ateapi.v1.ResourceMetadata + (*Selector)(nil), // 10: substrate.ateapi.v1.Selector + (*Container)(nil), // 11: substrate.ateapi.v1.Container + (*Volume)(nil), // 12: substrate.ateapi.v1.Volume + (*ObjectRef)(nil), // 13: substrate.ateapi.v1.ObjectRef + (*DeleteOptions)(nil), // 14: substrate.ateapi.v1.DeleteOptions +} +var file_template_proto_depIdxs = []int32{ + 9, // 0: substrate.ateapi.v1.ActorTemplate.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 10, // 1: substrate.ateapi.v1.ActorTemplate.worker_selector:type_name -> substrate.ateapi.v1.Selector + 11, // 2: substrate.ateapi.v1.ActorTemplate.containers:type_name -> substrate.ateapi.v1.Container + 12, // 3: substrate.ateapi.v1.ActorTemplate.volumes:type_name -> substrate.ateapi.v1.Volume + 4, // 4: substrate.ateapi.v1.ActorTemplate.snapshot_config:type_name -> substrate.ateapi.v1.SnapshotConfig + 2, // 5: substrate.ateapi.v1.ActorTemplate.resources:type_name -> substrate.ateapi.v1.ResourceRequirements + 3, // 6: substrate.ateapi.v1.ResourceRequirements.limits:type_name -> substrate.ateapi.v1.Resources + 5, // 7: substrate.ateapi.v1.SnapshotConfig.golden_snapshot_config:type_name -> substrate.ateapi.v1.GoldenSnapshotConfig + 0, // 8: substrate.ateapi.v1.GoldenSnapshotConfig.mode:type_name -> substrate.ateapi.v1.GoldenSnapshotMode + 1, // 9: substrate.ateapi.v1.CreateActorTemplateRequest.actor_template:type_name -> substrate.ateapi.v1.ActorTemplate + 13, // 10: substrate.ateapi.v1.GetActorTemplateRequest.actor_template:type_name -> substrate.ateapi.v1.ObjectRef + 13, // 11: substrate.ateapi.v1.DeleteActorTemplateRequest.actor_template:type_name -> substrate.ateapi.v1.ObjectRef + 14, // 12: substrate.ateapi.v1.DeleteActorTemplateRequest.options:type_name -> substrate.ateapi.v1.DeleteOptions + 13, // [13:13] is the sub-list for method output_type + 13, // [13:13] is the sub-list for method input_type + 13, // [13:13] is the sub-list for extension type_name + 13, // [13:13] is the sub-list for extension extendee + 0, // [0:13] is the sub-list for field type_name +} + +func init() { file_template_proto_init() } +func file_template_proto_init() { + if File_template_proto != nil { + return + } + file_common_proto_init() + file_container_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_template_proto_rawDesc), len(file_template_proto_rawDesc)), + NumEnums: 1, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_template_proto_goTypes, + DependencyIndexes: file_template_proto_depIdxs, + EnumInfos: file_template_proto_enumTypes, + MessageInfos: file_template_proto_msgTypes, + }.Build() + File_template_proto = out.File + file_template_proto_goTypes = nil + file_template_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/template.proto b/src/crux/sandbox/substrateapi/template.proto new file mode 100644 index 0000000000..adcde1ee59 --- /dev/null +++ b/src/crux/sandbox/substrateapi/template.proto @@ -0,0 +1,76 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "common.proto"; +import "container.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message ActorTemplate { + ResourceMetadata metadata = 1; + Selector worker_selector = 2; + repeated Container containers = 3; + repeated Volume volumes = 4; + SnapshotConfig snapshot_config = 5; + ResourceRequirements resources = 6; +} + +message ResourceRequirements { + Resources limits = 1; +} + +message Resources { + int64 cpu_millicores = 1; + int64 memory_bytes = 2; +} + +message SnapshotConfig { + GoldenSnapshotConfig golden_snapshot_config = 3; +} + +enum GoldenSnapshotMode { + GOLDEN_SNAPSHOT_MODE_UNSPECIFIED = 0; + GOLDEN_SNAPSHOT_MODE_ENABLED = 1; + GOLDEN_SNAPSHOT_MODE_DISABLED = 2; +} + +message GoldenSnapshotConfig { + GoldenSnapshotMode mode = 1; +} + +message CreateActorTemplateRequest { + ActorTemplate actor_template = 1; +} + +message GetActorTemplateRequest { + ObjectRef actor_template = 1; +} + +message DeleteActorTemplateRequest { + ObjectRef actor_template = 1; + DeleteOptions options = 2; +} diff --git a/src/crux/sandbox/substrateapi/worker.pb.go b/src/crux/sandbox/substrateapi/worker.pb.go new file mode 100644 index 0000000000..07611b1abc --- /dev/null +++ b/src/crux/sandbox/substrateapi/worker.pb.go @@ -0,0 +1,499 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: worker.proto + +package substrateapi + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type WorkerState int32 + +const ( + WorkerState_WORKER_STATE_UNSPECIFIED WorkerState = 0 + WorkerState_WORKER_STATE_ACTIVE WorkerState = 1 + WorkerState_WORKER_STATE_DRAINING WorkerState = 2 + WorkerState_WORKER_STATE_UNAVAILABLE WorkerState = 3 +) + +// Enum value maps for WorkerState. +var ( + WorkerState_name = map[int32]string{ + 0: "WORKER_STATE_UNSPECIFIED", + 1: "WORKER_STATE_ACTIVE", + 2: "WORKER_STATE_DRAINING", + 3: "WORKER_STATE_UNAVAILABLE", + } + WorkerState_value = map[string]int32{ + "WORKER_STATE_UNSPECIFIED": 0, + "WORKER_STATE_ACTIVE": 1, + "WORKER_STATE_DRAINING": 2, + "WORKER_STATE_UNAVAILABLE": 3, + } +) + +func (x WorkerState) Enum() *WorkerState { + p := new(WorkerState) + *p = x + return p +} + +func (x WorkerState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (WorkerState) Descriptor() protoreflect.EnumDescriptor { + return file_worker_proto_enumTypes[0].Descriptor() +} + +func (WorkerState) Type() protoreflect.EnumType { + return &file_worker_proto_enumTypes[0] +} + +func (x WorkerState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use WorkerState.Descriptor instead. +func (WorkerState) EnumDescriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{0} +} + +type Worker struct { + state protoimpl.MessageState `protogen:"open.v1"` + Metadata *ResourceMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + WorkerNamespace string `protobuf:"bytes,2,opt,name=worker_namespace,json=workerNamespace,proto3" json:"worker_namespace,omitempty"` + WorkerPool string `protobuf:"bytes,3,opt,name=worker_pool,json=workerPool,proto3" json:"worker_pool,omitempty"` + WorkerPod string `protobuf:"bytes,4,opt,name=worker_pod,json=workerPod,proto3" json:"worker_pod,omitempty"` + WorkerPodUid string `protobuf:"bytes,5,opt,name=worker_pod_uid,json=workerPodUid,proto3" json:"worker_pod_uid,omitempty"` + NodeName string `protobuf:"bytes,6,opt,name=node_name,json=nodeName,proto3" json:"node_name,omitempty"` + WorkerSpiffeId string `protobuf:"bytes,7,opt,name=worker_spiffe_id,json=workerSpiffeId,proto3" json:"worker_spiffe_id,omitempty"` + Labels map[string]string `protobuf:"bytes,8,rep,name=labels,proto3" json:"labels,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + Status *WorkerStatus `protobuf:"bytes,9,opt,name=status,proto3" json:"status,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Worker) Reset() { + *x = Worker{} + mi := &file_worker_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Worker) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Worker) ProtoMessage() {} + +func (x *Worker) ProtoReflect() protoreflect.Message { + mi := &file_worker_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Worker.ProtoReflect.Descriptor instead. +func (*Worker) Descriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{0} +} + +func (x *Worker) GetMetadata() *ResourceMetadata { + if x != nil { + return x.Metadata + } + return nil +} + +func (x *Worker) GetWorkerNamespace() string { + if x != nil { + return x.WorkerNamespace + } + return "" +} + +func (x *Worker) GetWorkerPool() string { + if x != nil { + return x.WorkerPool + } + return "" +} + +func (x *Worker) GetWorkerPod() string { + if x != nil { + return x.WorkerPod + } + return "" +} + +func (x *Worker) GetWorkerPodUid() string { + if x != nil { + return x.WorkerPodUid + } + return "" +} + +func (x *Worker) GetNodeName() string { + if x != nil { + return x.NodeName + } + return "" +} + +func (x *Worker) GetWorkerSpiffeId() string { + if x != nil { + return x.WorkerSpiffeId + } + return "" +} + +func (x *Worker) GetLabels() map[string]string { + if x != nil { + return x.Labels + } + return nil +} + +func (x *Worker) GetStatus() *WorkerStatus { + if x != nil { + return x.Status + } + return nil +} + +type WorkerStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + State WorkerState `protobuf:"varint,1,opt,name=state,proto3,enum=substrate.ateapi.v1.WorkerState" json:"state,omitempty"` + SandboxClasses []*WorkerSandboxClass `protobuf:"bytes,5,rep,name=sandbox_classes,json=sandboxClasses,proto3" json:"sandbox_classes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WorkerStatus) Reset() { + *x = WorkerStatus{} + mi := &file_worker_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WorkerStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WorkerStatus) ProtoMessage() {} + +func (x *WorkerStatus) ProtoReflect() protoreflect.Message { + mi := &file_worker_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WorkerStatus.ProtoReflect.Descriptor instead. +func (*WorkerStatus) Descriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{1} +} + +func (x *WorkerStatus) GetState() WorkerState { + if x != nil { + return x.State + } + return WorkerState_WORKER_STATE_UNSPECIFIED +} + +func (x *WorkerStatus) GetSandboxClasses() []*WorkerSandboxClass { + if x != nil { + return x.SandboxClasses + } + return nil +} + +type WorkerSandboxClass struct { + state protoimpl.MessageState `protogen:"open.v1"` + SandboxClass string `protobuf:"bytes,1,opt,name=sandbox_class,json=sandboxClass,proto3" json:"sandbox_class,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *WorkerSandboxClass) Reset() { + *x = WorkerSandboxClass{} + mi := &file_worker_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *WorkerSandboxClass) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WorkerSandboxClass) ProtoMessage() {} + +func (x *WorkerSandboxClass) ProtoReflect() protoreflect.Message { + mi := &file_worker_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WorkerSandboxClass.ProtoReflect.Descriptor instead. +func (*WorkerSandboxClass) Descriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{2} +} + +func (x *WorkerSandboxClass) GetSandboxClass() string { + if x != nil { + return x.SandboxClass + } + return "" +} + +type ListWorkersRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Options *ListOptions `protobuf:"bytes,1,opt,name=options,proto3" json:"options,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListWorkersRequest) Reset() { + *x = ListWorkersRequest{} + mi := &file_worker_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListWorkersRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListWorkersRequest) ProtoMessage() {} + +func (x *ListWorkersRequest) ProtoReflect() protoreflect.Message { + mi := &file_worker_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListWorkersRequest.ProtoReflect.Descriptor instead. +func (*ListWorkersRequest) Descriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{3} +} + +func (x *ListWorkersRequest) GetOptions() *ListOptions { + if x != nil { + return x.Options + } + return nil +} + +type ListWorkersResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + Workers []*Worker `protobuf:"bytes,1,rep,name=workers,proto3" json:"workers,omitempty"` + NextPageToken string `protobuf:"bytes,2,opt,name=next_page_token,json=nextPageToken,proto3" json:"next_page_token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ListWorkersResponse) Reset() { + *x = ListWorkersResponse{} + mi := &file_worker_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ListWorkersResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ListWorkersResponse) ProtoMessage() {} + +func (x *ListWorkersResponse) ProtoReflect() protoreflect.Message { + mi := &file_worker_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ListWorkersResponse.ProtoReflect.Descriptor instead. +func (*ListWorkersResponse) Descriptor() ([]byte, []int) { + return file_worker_proto_rawDescGZIP(), []int{4} +} + +func (x *ListWorkersResponse) GetWorkers() []*Worker { + if x != nil { + return x.Workers + } + return nil +} + +func (x *ListWorkersResponse) GetNextPageToken() string { + if x != nil { + return x.NextPageToken + } + return "" +} + +var File_worker_proto protoreflect.FileDescriptor + +const file_worker_proto_rawDesc = "" + + "\n" + + "\fworker.proto\x12\x13substrate.ateapi.v1\x1a\fcommon.proto\"\xda\x03\n" + + "\x06Worker\x12A\n" + + "\bmetadata\x18\x01 \x01(\v2%.substrate.ateapi.v1.ResourceMetadataR\bmetadata\x12)\n" + + "\x10worker_namespace\x18\x02 \x01(\tR\x0fworkerNamespace\x12\x1f\n" + + "\vworker_pool\x18\x03 \x01(\tR\n" + + "workerPool\x12\x1d\n" + + "\n" + + "worker_pod\x18\x04 \x01(\tR\tworkerPod\x12$\n" + + "\x0eworker_pod_uid\x18\x05 \x01(\tR\fworkerPodUid\x12\x1b\n" + + "\tnode_name\x18\x06 \x01(\tR\bnodeName\x12(\n" + + "\x10worker_spiffe_id\x18\a \x01(\tR\x0eworkerSpiffeId\x12?\n" + + "\x06labels\x18\b \x03(\v2'.substrate.ateapi.v1.Worker.LabelsEntryR\x06labels\x129\n" + + "\x06status\x18\t \x01(\v2!.substrate.ateapi.v1.WorkerStatusR\x06status\x1a9\n" + + "\vLabelsEntry\x12\x10\n" + + "\x03key\x18\x01 \x01(\tR\x03key\x12\x14\n" + + "\x05value\x18\x02 \x01(\tR\x05value:\x028\x01\"\x98\x01\n" + + "\fWorkerStatus\x126\n" + + "\x05state\x18\x01 \x01(\x0e2 .substrate.ateapi.v1.WorkerStateR\x05state\x12P\n" + + "\x0fsandbox_classes\x18\x05 \x03(\v2'.substrate.ateapi.v1.WorkerSandboxClassR\x0esandboxClasses\"9\n" + + "\x12WorkerSandboxClass\x12#\n" + + "\rsandbox_class\x18\x01 \x01(\tR\fsandboxClass\"P\n" + + "\x12ListWorkersRequest\x12:\n" + + "\aoptions\x18\x01 \x01(\v2 .substrate.ateapi.v1.ListOptionsR\aoptions\"t\n" + + "\x13ListWorkersResponse\x125\n" + + "\aworkers\x18\x01 \x03(\v2\x1b.substrate.ateapi.v1.WorkerR\aworkers\x12&\n" + + "\x0fnext_page_token\x18\x02 \x01(\tR\rnextPageToken*}\n" + + "\vWorkerState\x12\x1c\n" + + "\x18WORKER_STATE_UNSPECIFIED\x10\x00\x12\x17\n" + + "\x13WORKER_STATE_ACTIVE\x10\x01\x12\x19\n" + + "\x15WORKER_STATE_DRAINING\x10\x02\x12\x1c\n" + + "\x18WORKER_STATE_UNAVAILABLE\x10\x03BDZBgithub.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapib\x06proto3" + +var ( + file_worker_proto_rawDescOnce sync.Once + file_worker_proto_rawDescData []byte +) + +func file_worker_proto_rawDescGZIP() []byte { + file_worker_proto_rawDescOnce.Do(func() { + file_worker_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_worker_proto_rawDesc), len(file_worker_proto_rawDesc))) + }) + return file_worker_proto_rawDescData +} + +var file_worker_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_worker_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_worker_proto_goTypes = []any{ + (WorkerState)(0), // 0: substrate.ateapi.v1.WorkerState + (*Worker)(nil), // 1: substrate.ateapi.v1.Worker + (*WorkerStatus)(nil), // 2: substrate.ateapi.v1.WorkerStatus + (*WorkerSandboxClass)(nil), // 3: substrate.ateapi.v1.WorkerSandboxClass + (*ListWorkersRequest)(nil), // 4: substrate.ateapi.v1.ListWorkersRequest + (*ListWorkersResponse)(nil), // 5: substrate.ateapi.v1.ListWorkersResponse + nil, // 6: substrate.ateapi.v1.Worker.LabelsEntry + (*ResourceMetadata)(nil), // 7: substrate.ateapi.v1.ResourceMetadata + (*ListOptions)(nil), // 8: substrate.ateapi.v1.ListOptions +} +var file_worker_proto_depIdxs = []int32{ + 7, // 0: substrate.ateapi.v1.Worker.metadata:type_name -> substrate.ateapi.v1.ResourceMetadata + 6, // 1: substrate.ateapi.v1.Worker.labels:type_name -> substrate.ateapi.v1.Worker.LabelsEntry + 2, // 2: substrate.ateapi.v1.Worker.status:type_name -> substrate.ateapi.v1.WorkerStatus + 0, // 3: substrate.ateapi.v1.WorkerStatus.state:type_name -> substrate.ateapi.v1.WorkerState + 3, // 4: substrate.ateapi.v1.WorkerStatus.sandbox_classes:type_name -> substrate.ateapi.v1.WorkerSandboxClass + 8, // 5: substrate.ateapi.v1.ListWorkersRequest.options:type_name -> substrate.ateapi.v1.ListOptions + 1, // 6: substrate.ateapi.v1.ListWorkersResponse.workers:type_name -> substrate.ateapi.v1.Worker + 7, // [7:7] is the sub-list for method output_type + 7, // [7:7] is the sub-list for method input_type + 7, // [7:7] is the sub-list for extension type_name + 7, // [7:7] is the sub-list for extension extendee + 0, // [0:7] is the sub-list for field type_name +} + +func init() { file_worker_proto_init() } +func file_worker_proto_init() { + if File_worker_proto != nil { + return + } + file_common_proto_init() + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_worker_proto_rawDesc), len(file_worker_proto_rawDesc)), + NumEnums: 1, + NumMessages: 6, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_worker_proto_goTypes, + DependencyIndexes: file_worker_proto_depIdxs, + EnumInfos: file_worker_proto_enumTypes, + MessageInfos: file_worker_proto_msgTypes, + }.Build() + File_worker_proto = out.File + file_worker_proto_goTypes = nil + file_worker_proto_depIdxs = nil +} diff --git a/src/crux/sandbox/substrateapi/worker.proto b/src/crux/sandbox/substrateapi/worker.proto new file mode 100644 index 0000000000..13a527149e --- /dev/null +++ b/src/crux/sandbox/substrateapi/worker.proto @@ -0,0 +1,67 @@ +// Copyright 2026 The Agent Substrate Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Trimmed subset of api/ateapi/v1/ateapi.proto from +// https://github.com/agent-substrate/substrate (tag v1.0.0). Package, service, +// message names and field numbers match upstream so the generated client is +// wire-compatible with ate-api-server; only the RPCs and fields hive's +// Substrate sandbox launcher uses are kept, split across files so each +// generated *.pb.go stays under the crux 800-line file ratchet. Importing +// go.ate.dev/substrate directly would pull a go 1.27 toolchain requirement and +// a vendored cloud-SDK tree into crux. Regenerate with ./generate.sh. + +syntax = "proto3"; + +package substrate.ateapi.v1; + +import "common.proto"; + +option go_package = "github.com/hivecommons/hive/crux/sandbox/substrateapi;substrateapi"; + +message Worker { + ResourceMetadata metadata = 1; + string worker_namespace = 2; + string worker_pool = 3; + string worker_pod = 4; + string worker_pod_uid = 5; + string node_name = 6; + string worker_spiffe_id = 7; + map labels = 8; + WorkerStatus status = 9; +} + +enum WorkerState { + WORKER_STATE_UNSPECIFIED = 0; + WORKER_STATE_ACTIVE = 1; + WORKER_STATE_DRAINING = 2; + WORKER_STATE_UNAVAILABLE = 3; +} + +message WorkerStatus { + WorkerState state = 1; + repeated WorkerSandboxClass sandbox_classes = 5; +} + +message WorkerSandboxClass { + string sandbox_class = 1; +} + +message ListWorkersRequest { + ListOptions options = 1; +} + +message ListWorkersResponse { + repeated Worker workers = 1; + string next_page_token = 2; +} diff --git a/src/crux/sandbox/substratebridge/bridge.go b/src/crux/sandbox/substratebridge/bridge.go new file mode 100644 index 0000000000..498b1aaa62 --- /dev/null +++ b/src/crux/sandbox/substratebridge/bridge.go @@ -0,0 +1,439 @@ +// Package substratebridge is the in-actor side of hive's Agent Substrate +// runtime (ADR-0026 phase 2). Substrate has no exec, attach or logs RPC and +// cannot mount an existing PersistentVolumeClaim, so the sandbox image runs +// this small HTTP server as the actor's entrypoint and hive drives the kick +// through it over the atenet-router mTLS CONNECT tunnel: push the workspace +// in, run the command, pull the workspace back out. The server listens on +// the actor's private network only; the router is the only way in and it +// admits only allowlisted SPIFFE identities. +package substratebridge + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "strings" + "sync" + "time" +) + +const ( + // DefaultPort is the port the bridge serves on inside the actor and the + // port hive's CONNECT tunnel targets. + DefaultPort = 8477 + // DefaultWorkspace is where the bridge unpacks the kick workspace. + DefaultWorkspace = "/workspace" + + PathHealth = "/healthz" + PathWorkspace = "/workspace" + PathRun = "/run" + + // MaxOutput caps each of stdout and stderr in a RunResponse; the same + // 16 MiB the OpenShell launcher keeps. + MaxOutput = 16 << 20 + + // ExitCodeUnknown is reported when the command could not be started or + // ended without an exit status. + ExitCodeUnknown = -1 + + contentTypeTar = "application/x-tar+gzip" + contentTypeJSON = "application/json" + maxRequestJSON = 1 << 20 + maxTarEntry = 1 << 30 +) + +// RunRequest is the body of POST /run. +type RunRequest struct { + Command []string `json:"command"` + Env map[string]string `json:"env,omitempty"` + WorkDir string `json:"workdir,omitempty"` + // Stream asks for the output as it is produced: the response is then a + // sequence of JSON RunResponse frames, every one but the last marked + // Partial and carrying only new output, the last carrying the exit + // code. A bridge that predates the field ignores it and answers with a + // single complete RunResponse, which a streaming client reads as a + // final frame, so either side can be upgraded first. + Stream bool `json:"stream,omitempty"` +} + +// RunResponse is the body POST /run answers with once the command exits, +// or one frame of a streamed answer (see RunRequest.Stream). +type RunResponse struct { + Stdout string `json:"stdout"` + Stderr string `json:"stderr"` + ExitCode int `json:"exit_code"` + Error string `json:"error,omitempty"` + // Partial marks a streamed output frame; its ExitCode means nothing. + Partial bool `json:"partial,omitempty"` +} + +// Handler serves the bridge protocol for one workspace directory. +type Handler struct { + // Workspace is the directory PUT /workspace fills, GET /workspace packs + // and POST /run runs in. Defaults to DefaultWorkspace. + Workspace string + // RunTimeout bounds one POST /run; zero means the request context alone. + RunTimeout time.Duration +} + +// ServeHTTP implements http.Handler. +func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { + switch { + case r.URL.Path == PathHealth && r.Method == http.MethodGet: + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "ok\n") + case r.URL.Path == PathWorkspace && r.Method == http.MethodPut: + h.putWorkspace(w, r) + case r.URL.Path == PathWorkspace && r.Method == http.MethodGet: + h.getWorkspace(w, r) + case r.URL.Path == PathRun && r.Method == http.MethodPost: + h.run(w, r) + default: + http.Error(w, "substrate bridge: unknown route", http.StatusNotFound) + } +} + +func (h *Handler) workspace() string { + if h.Workspace != "" { + return h.Workspace + } + return DefaultWorkspace +} + +// clearDir empties dir without removing dir itself: inside the actor the +// workspace is a durableDir mount point, and unlinking a mount point fails +// with EBUSY. +func clearDir(dir string) error { + entries, err := os.ReadDir(dir) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + for _, e := range entries { + if err := os.RemoveAll(filepath.Join(dir, e.Name())); err != nil { + return err + } + } + return nil +} + +func (h *Handler) putWorkspace(w http.ResponseWriter, r *http.Request) { + dir := h.workspace() + if err := clearDir(dir); err != nil { + http.Error(w, fmt.Sprintf("substrate bridge: clearing %s: %v", dir, err), http.StatusInternalServerError) + return + } + if err := Unpack(r.Body, dir); err != nil { + http.Error(w, fmt.Sprintf("substrate bridge: unpacking workspace: %v", err), http.StatusBadRequest) + return + } + w.WriteHeader(http.StatusNoContent) +} + +func (h *Handler) getWorkspace(w http.ResponseWriter, _ *http.Request) { + var buf bytes.Buffer + if err := Pack(h.workspace(), &buf); err != nil { + http.Error(w, fmt.Sprintf("substrate bridge: packing workspace: %v", err), http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", contentTypeTar) + w.WriteHeader(http.StatusOK) + _, _ = io.Copy(w, &buf) +} + +func (h *Handler) run(w http.ResponseWriter, r *http.Request) { + var req RunRequest + if err := json.NewDecoder(io.LimitReader(r.Body, maxRequestJSON)).Decode(&req); err != nil { + http.Error(w, fmt.Sprintf("substrate bridge: decoding run request: %v", err), http.StatusBadRequest) + return + } + if len(req.Command) == 0 { + http.Error(w, "substrate bridge: command is required", http.StatusBadRequest) + return + } + ctx := r.Context() + if h.RunTimeout > 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, h.RunTimeout) + defer cancel() + } + w.Header().Set("Content-Type", contentTypeJSON) + if !req.Stream { + resp := Run(ctx, req, h.workspace()) + w.WriteHeader(http.StatusOK) + _ = json.NewEncoder(w).Encode(resp) + return + } + w.WriteHeader(http.StatusOK) + frames := &frameEncoder{enc: json.NewEncoder(w)} + if f, ok := w.(http.Flusher); ok { + frames.flush = f.Flush + } + resp := run(ctx, req, h.workspace(), &frameWriter{frames: frames, stderr: false}, &frameWriter{frames: frames, stderr: true}) + frames.send(RunResponse{ExitCode: resp.ExitCode, Error: resp.Error}) +} + +// frameEncoder serializes the streamed frames of one POST /run; stdout and +// stderr are written from separate goroutines by os/exec. +type frameEncoder struct { + mu sync.Mutex + enc *json.Encoder + flush func() +} + +func (f *frameEncoder) send(frame RunResponse) { + f.mu.Lock() + defer f.mu.Unlock() + _ = f.enc.Encode(frame) + if f.flush != nil { + f.flush() + } +} + +// frameWriter turns one of the command's output streams into Partial +// frames, capped at MaxOutput like the buffered answer. It never fails the +// command: a client that went away only loses the rest of the output. +type frameWriter struct { + frames *frameEncoder + stderr bool + written int +} + +func (w *frameWriter) Write(p []byte) (int, error) { + chunk := p + if room := MaxOutput - w.written; len(chunk) > room { + chunk = chunk[:max(room, 0)] + } + if len(chunk) > 0 { + w.written += len(chunk) + frame := RunResponse{Partial: true} + if w.stderr { + frame.Stderr = string(chunk) + } else { + frame.Stdout = string(chunk) + } + w.frames.send(frame) + } + return len(p), nil +} + +// Run executes one RunRequest with the request env replacing the process +// env — the launcher already filtered it through hive's credential-free +// allowlist — and reports the exit code the way the Podman launcher does. +func Run(ctx context.Context, req RunRequest, workspace string) RunResponse { + var stdout, stderr bytes.Buffer + resp := run(ctx, req, workspace, &stdout, &stderr) + resp.Stdout, resp.Stderr = capString(stdout.Bytes()), capString(stderr.Bytes()) + return resp +} + +// run executes req with its output going to stdout and stderr and reports +// the exit code; the output fields of the result are left empty. +func run(ctx context.Context, req RunRequest, workspace string, stdout, stderr io.Writer) RunResponse { + cmd := exec.CommandContext(ctx, req.Command[0], req.Command[1:]...) + cmd.Dir = req.WorkDir + if cmd.Dir == "" { + cmd.Dir = workspace + } + keys := make([]string, 0, len(req.Env)) + for k := range req.Env { + keys = append(keys, k) + } + sort.Strings(keys) + env := make([]string, 0, len(keys)+1) + for _, k := range keys { + env = append(env, k+"="+req.Env[k]) + } + if _, ok := req.Env["PATH"]; !ok { + env = append(env, "PATH="+os.Getenv("PATH")) + } + cmd.Env = env + cmd.Stdout, cmd.Stderr = stdout, stderr + err := cmd.Run() + var resp RunResponse + var exitErr *exec.ExitError + switch { + case err == nil: + case errors.As(err, &exitErr): + resp.ExitCode = exitErr.ExitCode() + default: + resp.ExitCode = ExitCodeUnknown + resp.Error = err.Error() + } + if ctx.Err() != nil && resp.Error == "" { + resp.Error = ctx.Err().Error() + } + return resp +} + +func capString(b []byte) string { + if len(b) > MaxOutput { + b = b[:MaxOutput] + } + return string(b) +} + +// Pack writes dir as a gzip-compressed tar stream: regular files, directories +// and symlinks, paths relative to dir. Git workspaces are small enough that a +// single stream is simpler than any chunked protocol. +func Pack(dir string, w io.Writer) error { + gz := gzip.NewWriter(w) + tw := tar.NewWriter(gz) + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + rel, err := filepath.Rel(dir, path) + if err != nil { + return err + } + if rel == "." { + return nil + } + link := "" + if info.Mode()&os.ModeSymlink != 0 { + if link, err = os.Readlink(path); err != nil { + return err + } + } + hdr, err := tar.FileInfoHeader(info, link) + if err != nil { + return err + } + hdr.Name = filepath.ToSlash(rel) + if info.IsDir() { + hdr.Name += "/" + } + if err := tw.WriteHeader(hdr); err != nil { + return err + } + if !info.Mode().IsRegular() { + return nil + } + f, err := os.Open(path) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(tw, f) + return err + }) + if err != nil { + return err + } + if err := tw.Close(); err != nil { + return err + } + return gz.Close() +} + +// Unpack reads a Pack stream into dir, refusing entries that escape it +// lexically (absolute or ".." names) or through symlinks: every write goes +// through an os.Root, so a symlink entry may be created but nothing is ever +// written through one that leaves dir. +func Unpack(r io.Reader, dir string) error { + gz, err := gzip.NewReader(r) + if err != nil { + return fmt.Errorf("gzip: %w", err) + } + defer gz.Close() + if err := os.MkdirAll(dir, 0o755); err != nil { + return err + } + root, err := os.OpenRoot(dir) + if err != nil { + return err + } + defer root.Close() + tr := tar.NewReader(gz) + for { + hdr, err := tr.Next() + if errors.Is(err, io.EOF) { + return nil + } + if err != nil { + return fmt.Errorf("tar: %w", err) + } + rel, err := safeRel(hdr.Name) + if err != nil { + return err + } + if err := unpackEntry(tr, hdr, root, rel); err != nil { + return fmt.Errorf("tar: entry %q: %w", hdr.Name, err) + } + } +} + +func unpackEntry(tr *tar.Reader, hdr *tar.Header, root *os.Root, rel string) error { + switch hdr.Typeflag { + case tar.TypeDir: + return root.MkdirAll(rel, os.FileMode(hdr.Mode)&os.ModePerm|0o700) + case tar.TypeSymlink: + if err := root.MkdirAll(filepath.Dir(rel), 0o755); err != nil { + return err + } + _ = root.Remove(rel) + return root.Symlink(hdr.Linkname, rel) + case tar.TypeReg: + if hdr.Size > maxTarEntry { + return fmt.Errorf("tar: entry %q is too large (%d bytes)", hdr.Name, hdr.Size) + } + if err := root.MkdirAll(filepath.Dir(rel), 0o755); err != nil { + return err + } + f, err := root.OpenFile(rel, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(hdr.Mode)&os.ModePerm|0o600) + if err != nil { + return err + } + if _, err := io.Copy(f, io.LimitReader(tr, hdr.Size)); err != nil { + f.Close() + return err + } + return f.Close() + default: + // Devices, FIFOs and hard links have no place in a git workspace. + return fmt.Errorf("tar: entry %q has unsupported type %q", hdr.Name, hdr.Typeflag) + } +} + +// safeRel is the lexical first filter; os.Root enforces the rest. +func safeRel(name string) (string, error) { + clean := filepath.Clean(filepath.FromSlash(name)) + if filepath.IsAbs(clean) || clean == ".." || strings.HasPrefix(clean, ".."+string(filepath.Separator)) { + return "", fmt.Errorf("tar: entry %q escapes the workspace", name) + } + return clean, nil +} + +// Serve runs the bridge on ln until ctx ends. The actor's command is +// expected to be the sandbox image's bridge entrypoint calling this. +func Serve(ctx context.Context, ln net.Listener, h *Handler) error { + srv := &http.Server{Handler: h, ReadHeaderTimeout: 10 * time.Second} + errCh := make(chan error, 1) + go func() { errCh <- srv.Serve(ln) }() + select { + case <-ctx.Done(): + shutdownCtx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + _ = srv.Shutdown(shutdownCtx) + return ctx.Err() + case err := <-errCh: + if errors.Is(err, http.ErrServerClosed) { + return nil + } + return err + } +} diff --git a/src/crux/sandbox/substratebridge/bridge_test.go b/src/crux/sandbox/substratebridge/bridge_test.go new file mode 100644 index 0000000000..c7e40bde59 --- /dev/null +++ b/src/crux/sandbox/substratebridge/bridge_test.go @@ -0,0 +1,525 @@ +package substratebridge + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "io" + "net" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" +) + +func writeFile(t *testing.T, path, content string) { + t.Helper() + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o644); err != nil { + t.Fatal(err) + } +} + +func TestPackUnpackRoundTrip(t *testing.T) { + src := t.TempDir() + writeFile(t, filepath.Join(src, "a.txt"), "alpha") + writeFile(t, filepath.Join(src, "sub", "deep", "b.txt"), "beta") + if err := os.Mkdir(filepath.Join(src, "empty"), 0o755); err != nil { + t.Fatal(err) + } + if err := os.Symlink("a.txt", filepath.Join(src, "link")); err != nil { + t.Fatal(err) + } + if err := os.Chmod(filepath.Join(src, "a.txt"), 0o755); err != nil { + t.Fatal(err) + } + + var buf bytes.Buffer + if err := Pack(src, &buf); err != nil { + t.Fatalf("Pack: %v", err) + } + dst := filepath.Join(t.TempDir(), "out") + if err := Unpack(&buf, dst); err != nil { + t.Fatalf("Unpack: %v", err) + } + if got, _ := os.ReadFile(filepath.Join(dst, "a.txt")); string(got) != "alpha" { + t.Errorf("a.txt = %q", got) + } + if got, _ := os.ReadFile(filepath.Join(dst, "sub", "deep", "b.txt")); string(got) != "beta" { + t.Errorf("b.txt = %q", got) + } + if st, err := os.Stat(filepath.Join(dst, "empty")); err != nil || !st.IsDir() { + t.Errorf("empty dir not restored: %v", err) + } + if target, err := os.Readlink(filepath.Join(dst, "link")); err != nil || target != "a.txt" { + t.Errorf("symlink = %q, %v", target, err) + } + if st, _ := os.Stat(filepath.Join(dst, "a.txt")); st.Mode()&0o100 == 0 { + t.Errorf("executable bit lost: %v", st.Mode()) + } +} + +func TestPackMissingDir(t *testing.T) { + if err := Pack(filepath.Join(t.TempDir(), "nope"), io.Discard); err == nil { + t.Fatal("Pack of a missing dir succeeded") + } +} + +func tarStream(t *testing.T, build func(tw *tar.Writer)) *bytes.Buffer { + t.Helper() + var buf bytes.Buffer + gz := gzip.NewWriter(&buf) + tw := tar.NewWriter(gz) + build(tw) + // A deliberately truncated entry (header without body) makes Close + // complain; the header is already on the wire, which is what the + // test wants. + _ = tw.Close() + if err := gz.Close(); err != nil { + t.Fatal(err) + } + return &buf +} + +func TestUnpackRejectsBadStreams(t *testing.T) { + dst := t.TempDir() + if err := Unpack(strings.NewReader("not gzip"), dst); err == nil || !strings.Contains(err.Error(), "gzip") { + t.Errorf("non-gzip accepted: %v", err) + } + escape := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "../evil", Typeflag: tar.TypeReg, Size: 0, Mode: 0o644}) + }) + if err := Unpack(escape, dst); err == nil || !strings.Contains(err.Error(), "escapes") { + t.Errorf("escaping entry accepted: %v", err) + } + abs := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "/etc/passwd", Typeflag: tar.TypeReg, Size: 0, Mode: 0o644}) + }) + if err := Unpack(abs, dst); err == nil || !strings.Contains(err.Error(), "escapes") { + t.Errorf("absolute entry accepted: %v", err) + } + fifo := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "pipe", Typeflag: tar.TypeFifo, Mode: 0o644}) + }) + if err := Unpack(fifo, dst); err == nil || !strings.Contains(err.Error(), "unsupported type") { + t.Errorf("fifo accepted: %v", err) + } + huge := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "big", Typeflag: tar.TypeReg, Size: maxTarEntry + 1, Mode: 0o644}) + }) + if err := Unpack(huge, dst); err == nil || !strings.Contains(err.Error(), "too large") { + t.Errorf("oversized entry accepted: %v", err) + } + // Truncated tar body: gzip is fine, tar is not. + var truncated bytes.Buffer + gz := gzip.NewWriter(&truncated) + _, _ = gz.Write([]byte("garbage that is not a tar header at all")) + _ = gz.Close() + if err := Unpack(&truncated, dst); err == nil || !strings.Contains(err.Error(), "tar") { + t.Errorf("garbage tar accepted: %v", err) + } + // Destination that cannot be created. + blocker := filepath.Join(t.TempDir(), "file") + writeFile(t, blocker, "x") + empty := tarStream(t, func(*tar.Writer) {}) + if err := Unpack(empty, filepath.Join(blocker, "sub")); err == nil { + t.Errorf("unpack under a regular file succeeded") + } +} + +func TestUnpackSymlinkReplacesExistingAndNestedEntries(t *testing.T) { + dst := t.TempDir() + writeFile(t, filepath.Join(dst, "link"), "a file where the link goes") + stream := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "link", Typeflag: tar.TypeSymlink, Linkname: "target", Mode: 0o777}) + _ = tw.WriteHeader(&tar.Header{Name: "d/e/link2", Typeflag: tar.TypeSymlink, Linkname: "../x", Mode: 0o777}) + _ = tw.WriteHeader(&tar.Header{Name: "d/e/f.txt", Typeflag: tar.TypeReg, Size: 2, Mode: 0o600}) + _, _ = tw.Write([]byte("hi")) + }) + if err := Unpack(stream, dst); err != nil { + t.Fatalf("Unpack: %v", err) + } + if target, err := os.Readlink(filepath.Join(dst, "link")); err != nil || target != "target" { + t.Errorf("link = %q, %v", target, err) + } + if got, _ := os.ReadFile(filepath.Join(dst, "d", "e", "f.txt")); string(got) != "hi" { + t.Errorf("f.txt = %q", got) + } +} + +func TestUnpackRefusesWriteThroughSymlink(t *testing.T) { + for name, linkname := range map[string]func(outside string) string{ + "absolute": func(outside string) string { return outside }, + "relative": func(string) string { return "../outside-rel" }, + } { + t.Run(name, func(t *testing.T) { + parent := t.TempDir() + dst := filepath.Join(parent, "ws") + outside := t.TempDir() + if name == "relative" { + outside = filepath.Join(parent, "outside-rel") + if err := os.Mkdir(outside, 0o755); err != nil { + t.Fatal(err) + } + } + stream := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "link", Typeflag: tar.TypeSymlink, Linkname: linkname(outside), Mode: 0o777}) + _ = tw.WriteHeader(&tar.Header{Name: "link/pwned", Typeflag: tar.TypeReg, Size: 2, Mode: 0o644}) + _, _ = tw.Write([]byte("no")) + }) + if err := Unpack(stream, dst); err == nil { + t.Fatal("write through escaping symlink succeeded") + } + if _, err := os.Stat(filepath.Join(outside, "pwned")); !os.IsNotExist(err) { + t.Errorf("file written outside the workspace: %v", err) + } + }) + } +} + +func TestUnpackEntryErrorsWhenParentIsAFile(t *testing.T) { + dst := t.TempDir() + writeFile(t, filepath.Join(dst, "blocker"), "x") + for _, hdr := range []*tar.Header{ + {Name: "blocker/sub/file", Typeflag: tar.TypeReg, Size: 0, Mode: 0o644}, + {Name: "blocker/sub/link", Typeflag: tar.TypeSymlink, Linkname: "x", Mode: 0o777}, + {Name: "blocker/sub/", Typeflag: tar.TypeDir, Mode: 0o755}, + } { + h := hdr + stream := tarStream(t, func(tw *tar.Writer) { _ = tw.WriteHeader(h) }) + if err := Unpack(stream, dst); err == nil { + t.Errorf("entry %q under a regular file succeeded", h.Name) + } + } + // A regular file whose target is an existing directory cannot be opened. + if err := os.Mkdir(filepath.Join(dst, "dir"), 0o755); err != nil { + t.Fatal(err) + } + stream := tarStream(t, func(tw *tar.Writer) { + _ = tw.WriteHeader(&tar.Header{Name: "dir", Typeflag: tar.TypeReg, Size: 0, Mode: 0o644}) + }) + if err := Unpack(stream, dst); err == nil { + t.Errorf("regular file over a directory succeeded") + } +} + +func TestRunReportsExitCodesAndErrors(t *testing.T) { + ws := t.TempDir() + ok := Run(context.Background(), RunRequest{Command: []string{"sh", "-c", "echo out; echo err >&2; pwd"}}, ws) + if ok.ExitCode != 0 || ok.Error != "" || !strings.Contains(ok.Stdout, "out") || !strings.Contains(ok.Stderr, "err") { + t.Errorf("ok run = %+v", ok) + } + if evalWS, _ := filepath.EvalSymlinks(ws); !strings.Contains(ok.Stdout, filepath.Base(evalWS)) { + t.Errorf("command did not run in the workspace: %q", ok.Stdout) + } + + fail := Run(context.Background(), RunRequest{Command: []string{"sh", "-c", "exit 7"}}, ws) + if fail.ExitCode != 7 || fail.Error != "" { + t.Errorf("exit 7 run = %+v", fail) + } + + missing := Run(context.Background(), RunRequest{Command: []string{"/definitely/not/a/binary"}}, ws) + if missing.ExitCode != ExitCodeUnknown || missing.Error == "" { + t.Errorf("missing binary run = %+v", missing) + } + + env := Run(context.Background(), RunRequest{Command: []string{"sh", "-c", "echo $FOO:$BAR; env | grep -c ^HOME= || true"}, Env: map[string]string{"FOO": "1", "BAR": "2"}}, ws) + if !strings.HasPrefix(env.Stdout, "1:2\n0") { + t.Errorf("env not replaced wholesale: %q", env.Stdout) + } + withPath := Run(context.Background(), RunRequest{Command: []string{"sh", "-c", "echo $PATH"}, Env: map[string]string{"PATH": "/custom/bin"}}, ws) + if strings.TrimSpace(withPath.Stdout) != "/custom/bin" { + t.Errorf("explicit PATH not honoured: %q", withPath.Stdout) + } + + other := t.TempDir() + wd := Run(context.Background(), RunRequest{Command: []string{"pwd"}, WorkDir: other}, ws) + if evalOther, _ := filepath.EvalSymlinks(other); !strings.Contains(wd.Stdout, filepath.Base(evalOther)) { + t.Errorf("workdir not honoured: %q", wd.Stdout) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + cancelled := Run(ctx, RunRequest{Command: []string{"sh", "-c", "exit 0"}}, ws) + if cancelled.Error == "" { + t.Errorf("cancelled run reported no error: %+v", cancelled) + } +} + +func TestCapString(t *testing.T) { + if got := capString(bytes.Repeat([]byte("x"), MaxOutput+10)); len(got) != MaxOutput { + t.Errorf("len = %d", len(got)) + } +} + +func newBridgeServer(t *testing.T) (*httptest.Server, string) { + t.Helper() + ws := filepath.Join(t.TempDir(), "ws") + srv := httptest.NewServer(&Handler{Workspace: ws}) + t.Cleanup(srv.Close) + return srv, ws +} + +func TestHandlerRoutes(t *testing.T) { + srv, ws := newBridgeServer(t) + + resp, err := http.Get(srv.URL + PathHealth) + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("healthz: %v %v", err, resp) + } + resp.Body.Close() + + src := t.TempDir() + writeFile(t, filepath.Join(src, "hello.txt"), "hi") + var buf bytes.Buffer + if err := Pack(src, &buf); err != nil { + t.Fatal(err) + } + req, _ := http.NewRequest(http.MethodPut, srv.URL+PathWorkspace, &buf) + resp, err = http.DefaultClient.Do(req) + if err != nil || resp.StatusCode != http.StatusNoContent { + t.Fatalf("put workspace: %v %v", err, resp) + } + resp.Body.Close() + if got, _ := os.ReadFile(filepath.Join(ws, "hello.txt")); string(got) != "hi" { + t.Errorf("workspace not unpacked: %q", got) + } + + body, _ := json.Marshal(RunRequest{Command: []string{"sh", "-c", "echo changed > hello.txt; exit 3"}}) + resp, err = http.Post(srv.URL+PathRun, contentTypeJSON, bytes.NewReader(body)) + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("run: %v %v", err, resp) + } + var out RunResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatal(err) + } + resp.Body.Close() + if out.ExitCode != 3 { + t.Errorf("run response = %+v", out) + } + + resp, err = http.Get(srv.URL + PathWorkspace) + if err != nil || resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Type") != contentTypeTar { + t.Fatalf("get workspace: %v %v", err, resp) + } + dst := t.TempDir() + if err := Unpack(resp.Body, dst); err != nil { + t.Fatal(err) + } + resp.Body.Close() + if got, _ := os.ReadFile(filepath.Join(dst, "hello.txt")); strings.TrimSpace(string(got)) != "changed" { + t.Errorf("pulled workspace = %q", got) + } +} + +func TestHandlerErrors(t *testing.T) { + srv, ws := newBridgeServer(t) + status := func(method, path string, body io.Reader) int { + req, _ := http.NewRequest(method, srv.URL+path, body) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + return resp.StatusCode + } + if got := status(http.MethodGet, "/nope", nil); got != http.StatusNotFound { + t.Errorf("unknown route = %d", got) + } + if got := status(http.MethodDelete, PathWorkspace, nil); got != http.StatusNotFound { + t.Errorf("unknown method = %d", got) + } + if got := status(http.MethodPut, PathWorkspace, strings.NewReader("not a tarball")); got != http.StatusBadRequest { + t.Errorf("bad tarball = %d", got) + } + if got := status(http.MethodPost, PathRun, strings.NewReader("{")); got != http.StatusBadRequest { + t.Errorf("bad json = %d", got) + } + if got := status(http.MethodPost, PathRun, strings.NewReader(`{"command":[]}`)); got != http.StatusBadRequest { + t.Errorf("empty command = %d", got) + } + // Workspace missing: GET cannot pack it. + if got := status(http.MethodGet, PathWorkspace, nil); got != http.StatusInternalServerError { + t.Errorf("get of missing workspace = %d", got) + } + // Workspace parent is a file: PUT cannot recreate it. + _ = ws + blocked := httptest.NewServer(&Handler{Workspace: filepath.Join(ws, "..", "ws-file", "sub")}) + defer blocked.Close() + writeFile(t, filepath.Join(ws, "..", "ws-file"), "x") + empty := tarStream(t, func(*tar.Writer) {}) + req, _ := http.NewRequest(http.MethodPut, blocked.URL+PathWorkspace, empty) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusInternalServerError { + t.Errorf("put under a file = %d, want 500 from clearing the workspace", resp.StatusCode) + } +} + +func TestHandlerDefaultsAndRunTimeout(t *testing.T) { + h := &Handler{} + if h.workspace() != DefaultWorkspace { + t.Errorf("default workspace = %q", h.workspace()) + } + // RunTimeout already expired: the command is killed and the response + // says why, without the test waiting on it. + srv := httptest.NewServer(&Handler{Workspace: t.TempDir(), RunTimeout: 1}) + defer srv.Close() + body, _ := json.Marshal(RunRequest{Command: []string{"sh", "-c", "sleep 30"}}) + resp, err := http.Post(srv.URL+PathRun, contentTypeJSON, bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + var out RunResponse + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatal(err) + } + if out.Error == "" { + t.Errorf("timed-out run reported no error: %+v", out) + } +} + +func TestServeStopsOnContextAndListenerClose(t *testing.T) { + ln, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- Serve(ctx, ln, &Handler{Workspace: t.TempDir()}) }() + resp, err := http.Get("http://" + ln.Addr().String() + PathHealth) + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("healthz over Serve: %v %v", err, resp) + } + resp.Body.Close() + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Errorf("Serve returned %v, want context.Canceled", err) + } + + ln2, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + done2 := make(chan error, 1) + go func() { done2 <- Serve(context.Background(), ln2, &Handler{}) }() + // Closing the listener makes Serve return the listener error (not + // ErrServerClosed, which only Shutdown produces). + _ = ln2.Close() + if err := <-done2; err == nil { + t.Errorf("Serve returned nil after the listener closed") + } +} + +// Inside the actor /workspace is a durableDir mount point; PUT must empty it +// in place rather than unlink it (which fails with EBUSY on a mount). +func TestPutWorkspaceClearsInPlace(t *testing.T) { + ws := t.TempDir() + writeFile(t, filepath.Join(ws, "stale.txt"), "old") + writeFile(t, filepath.Join(ws, "nested", "deep.txt"), "old") + before, err := os.Stat(ws) + if err != nil { + t.Fatal(err) + } + srv := httptest.NewServer(&Handler{Workspace: ws}) + defer srv.Close() + body := tarStream(t, func(tw *tar.Writer) { + content := []byte("new") + _ = tw.WriteHeader(&tar.Header{Name: "fresh.txt", Mode: 0o644, Size: int64(len(content)), Typeflag: tar.TypeReg}) + _, _ = tw.Write(content) + }) + req, _ := http.NewRequest(http.MethodPut, srv.URL+PathWorkspace, body) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + resp.Body.Close() + if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK { + t.Fatalf("put = %d", resp.StatusCode) + } + after, err := os.Stat(ws) + if err != nil { + t.Fatal(err) + } + if !os.SameFile(before, after) { + t.Error("workspace directory was recreated; it must be emptied in place") + } + if _, err := os.Stat(filepath.Join(ws, "stale.txt")); !os.IsNotExist(err) { + t.Error("stale.txt survived PUT") + } + if _, err := os.Stat(filepath.Join(ws, "nested")); !os.IsNotExist(err) { + t.Error("nested/ survived PUT") + } + if got, err := os.ReadFile(filepath.Join(ws, "fresh.txt")); err != nil || string(got) != "new" { + t.Errorf("fresh.txt = %q, %v", got, err) + } + if err := clearDir(filepath.Join(ws, "does-not-exist")); err != nil { + t.Errorf("clearDir of a missing dir = %v, want nil", err) + } +} + +func TestHandlerStreamsRunOutput(t *testing.T) { + srv, ws := newBridgeServer(t) + if err := os.MkdirAll(ws, 0o755); err != nil { + t.Fatal(err) + } + body, _ := json.Marshal(RunRequest{Command: []string{"sh", "-c", "echo one; echo two >&2; exit 2"}, Stream: true}) + resp, err := http.Post(srv.URL+PathRun, contentTypeJSON, bytes.NewReader(body)) + if err != nil || resp.StatusCode != http.StatusOK { + t.Fatalf("run: %v %v", err, resp) + } + defer resp.Body.Close() + dec := json.NewDecoder(resp.Body) + var stdout, stderr strings.Builder + var frames []RunResponse + for { + var f RunResponse + if err := dec.Decode(&f); errors.Is(err, io.EOF) { + break + } else if err != nil { + t.Fatal(err) + } + frames = append(frames, f) + stdout.WriteString(f.Stdout) + stderr.WriteString(f.Stderr) + } + if len(frames) < 2 { + t.Fatalf("frames = %+v", frames) + } + for _, f := range frames[:len(frames)-1] { + if !f.Partial { + t.Fatalf("only the last frame may be final: %+v", frames) + } + } + if last := frames[len(frames)-1]; last.Partial || last.ExitCode != 2 || last.Stdout != "" || last.Stderr != "" { + t.Fatalf("final frame = %+v", last) + } + if stdout.String() != "one\n" || stderr.String() != "two\n" { + t.Fatalf("stdout=%q stderr=%q", stdout.String(), stderr.String()) + } +} + +func TestFrameWriterCapsOutput(t *testing.T) { + var buf bytes.Buffer + w := &frameWriter{frames: &frameEncoder{enc: json.NewEncoder(&buf)}} + big := make([]byte, MaxOutput+5) + if n, err := w.Write(big); err != nil || n != len(big) { + t.Fatalf("Write = %d %v", n, err) + } + buf.Reset() + if n, err := w.Write([]byte("more")); err != nil || n != 4 || buf.Len() != 0 { + t.Fatalf("past the cap: n=%d err=%v frames=%q", n, err, buf.String()) + } +} diff --git a/src/crux/serve/metrics.go b/src/crux/serve/metrics.go index 4ea8b016c7..e21088a79c 100644 --- a/src/crux/serve/metrics.go +++ b/src/crux/serve/metrics.go @@ -26,6 +26,7 @@ const ( requestDurationMetric = "crux.serve.request.duration" requestCountMetric = "crux.serve.requests" + authRejectionMetric = "crux.serve.auth.rejections" routeUnmatched = "unmatched" @@ -38,16 +39,28 @@ const ( methodOther = "_OTHER" ) +// Why authenticate refused a request: a fixed enum, so the rejection counter's +// reason attribute stays bounded. +const ( + rejectProjectForbidden = "project_forbidden" + rejectNoTokenConfigured = "no_token_configured" + rejectInvalidToken = "missing_or_invalid_token" +) + var ( + attrReason = attribute.Key("reason") attrMethod = attribute.Key("method") attrRoute = attribute.Key("route") attrStatusClass = attribute.Key("status_class") ) -// requestMetrics holds the two serve request instruments. +// requestMetrics holds the serve request and auth rejection instruments. type requestMetrics struct { duration metric.Float64Histogram requests metric.Int64Counter + // rejections counts requests authenticate refused with a 401. They never + // reach instrument, which sits inside authenticate. + rejections metric.Int64Counter } func newRequestMetrics(meter metric.Meter) requestMetrics { @@ -58,7 +71,9 @@ func newRequestMetrics(meter metric.Meter) requestMetrics { metric.WithDescription("Time crux serve took to answer a request.")) requests, _ := meter.Int64Counter(requestCountMetric, metric.WithDescription("crux serve requests, by method, route pattern and status class.")) - return requestMetrics{duration: duration, requests: requests} + rejections, _ := meter.Int64Counter(authRejectionMetric, + metric.WithDescription("crux serve requests refused by authentication, by reason.")) + return requestMetrics{duration: duration, requests: requests, rejections: rejections} } var globalRequestMetrics = sync.OnceValue(func() requestMetrics { @@ -168,6 +183,11 @@ func instrument(next http.Handler) http.Handler { }) } +// recordAuthRejection counts one 401 with only the bounded reason attribute. +func recordAuthRejection(ctx context.Context, reason string) { + globalRequestMetrics().rejections.Add(ctx, 1, metric.WithAttributes(attrReason.String(reason))) +} + func (m requestMetrics) record(ctx context.Context, d time.Duration, attrs []attribute.KeyValue) { opt := metric.WithAttributes(attrs...) m.duration.Record(ctx, d.Seconds(), opt) diff --git a/src/crux/serve/metrics_test.go b/src/crux/serve/metrics_test.go index 823724d53a..256fbeb7ef 100644 --- a/src/crux/serve/metrics_test.go +++ b/src/crux/serve/metrics_test.go @@ -7,6 +7,8 @@ import ( "strings" "testing" + "github.com/hivecommons/hive/crux/api" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/codes" sdkmetric "go.opentelemetry.io/otel/sdk/metric" @@ -334,3 +336,42 @@ func collectStatusClasses(t *testing.T, reader *sdkmetric.ManualReader) []string } return classes } + +func TestAuthRejectionsCountedByReason(t *testing.T) { + reader := useTestMetrics(t) + _, h := principalFixture(t) + + callWith(h, api.PathHealth, "wrong-secret-token") + callWith(h, api.PathHealth, "") + callWith(h, api.PathHealth, testProjectToken) + callWith(h, api.PathHealth, "another-wrong-token") + + var rm metricdata.ResourceMetrics + if err := reader.Collect(context.Background(), &rm); err != nil { + t.Fatalf("Collect: %v", err) + } + got := map[string]int64{} + for _, sm := range rm.ScopeMetrics { + for _, m := range sm.Metrics { + if m.Name != authRejectionMetric { + continue + } + for _, dp := range m.Data.(metricdata.Sum[int64]).DataPoints { + if dp.Attributes.Len() != 1 { + t.Errorf("attributes = %v, want only reason", dp.Attributes) + } + reason, _ := dp.Attributes.Value(attrReason) + got[reason.AsString()] = dp.Value + } + } + } + want := map[string]int64{rejectInvalidToken: 3, rejectProjectForbidden: 1} + if len(got) != len(want) { + t.Fatalf("rejections = %v, want %v", got, want) + } + for k, v := range want { + if got[k] != v { + t.Errorf("rejections[%s] = %d, want %d", k, got[k], v) + } + } +} diff --git a/src/crux/serve/serve.go b/src/crux/serve/serve.go index bb0f4e8e83..99ddbc48f2 100644 --- a/src/crux/serve/serve.go +++ b/src/crux/serve/serve.go @@ -209,6 +209,7 @@ func (s *Server) authenticate(next http.Handler) http.Handler { token, _ := strings.CutPrefix(r.Header.Get("Authorization"), bearerPrefix) if project, projectToken := s.LaneAuth.Project(token); projectToken { if !s.isWorkService(r.URL.Path) { + recordAuthRejection(r.Context(), rejectProjectForbidden) s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "project credentials cannot access agent RPCs") return } @@ -217,6 +218,7 @@ func (s *Server) authenticate(next http.Handler) http.Handler { } if s.Token == "" { if network != networkUnix { + recordAuthRejection(r.Context(), rejectNoTokenConfigured) s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "this server has no bearer token; only unix-socket requests are accepted") return } @@ -225,6 +227,7 @@ func (s *Server) authenticate(next http.Handler) http.Handler { } got, ok := strings.CutPrefix(r.Header.Get("Authorization"), bearerPrefix) if !ok || subtle.ConstantTimeCompare([]byte(got), []byte(s.Token)) != 1 { + recordAuthRejection(r.Context(), rejectInvalidToken) w.Header().Set("WWW-Authenticate", `Bearer realm="crux"`) s.writeError(w, http.StatusUnauthorized, api.CodeUnauthorized, "missing or invalid bearer token") return diff --git a/src/crux/upstreamproxy/upstreamproxy.go b/src/crux/upstreamproxy/upstreamproxy.go index 723dfa230b..52abb11359 100644 --- a/src/crux/upstreamproxy/upstreamproxy.go +++ b/src/crux/upstreamproxy/upstreamproxy.go @@ -144,17 +144,24 @@ func parseProxyURL(raw string) (*url.URL, error) { if !strings.Contains(raw, "://") { raw = "http://" + raw } + // Errors are logged by the caller; the URL's userinfo is the proxy's + // Basic credential, so never echo raw. u, err := url.Parse(raw) if err != nil { - return nil, fmt.Errorf("upstream proxy url %q: %w", raw, err) + // *url.Error.Error() repeats the full URL; keep only its inner cause. + var ue *url.Error + if errors.As(err, &ue) { + err = ue.Err + } + return nil, fmt.Errorf("upstream proxy url %q: %w", redactUserinfo(raw), err) } switch u.Scheme { case "http", "https": default: - return nil, fmt.Errorf("upstream proxy url %q: scheme %q not supported (http or https)", raw, u.Scheme) + return nil, fmt.Errorf("upstream proxy url %q: scheme %q not supported (http or https)", u.Redacted(), u.Scheme) } if u.Hostname() == "" { - return nil, fmt.Errorf("upstream proxy url %q: missing host", raw) + return nil, fmt.Errorf("upstream proxy url %q: missing host", u.Redacted()) } if u.Port() == "" { port := "80" @@ -166,6 +173,21 @@ func parseProxyURL(raw string) (*url.URL, error) { return u, nil } +// redactUserinfo masks everything between "://" and the first "@" of raw, for +// URLs url.Parse rejected and so cannot be redacted structurally. +func redactUserinfo(raw string) string { + i := strings.Index(raw, "://") + if i < 0 { + return raw + } + rest := raw[i+3:] + at := strings.Index(rest, "@") + if at < 0 { + return raw + } + return raw[:i+3] + "xxxxx@" + rest[at+1:] +} + func envList(getenv func(string) string, name string) ([]string, bool) { raw := getenv(name) if strings.TrimSpace(raw) == "" { diff --git a/src/crux/upstreamproxy/upstreamproxy_test.go b/src/crux/upstreamproxy/upstreamproxy_test.go index f3d45b8623..5c7d84dcab 100644 --- a/src/crux/upstreamproxy/upstreamproxy_test.go +++ b/src/crux/upstreamproxy/upstreamproxy_test.go @@ -144,6 +144,27 @@ func TestResolveRejectsBadInput(t *testing.T) { } } +func TestResolveErrorsRedactUserinfo(t *testing.T) { + // The caller logs the error; the URL's userinfo is the proxy's Basic + // credential and must not reach the log through any rejection branch. + for _, raw := range []string{ + "socks5://alice:s3cret-pw@proxy.corp:1080", // unsupported scheme + "http://alice:s3cret-pw@", // missing host + "http://alice:s3cret-pw@proxy.corp:%zz", // url.Parse failure + } { + _, err := Resolve(Config{URL: raw}, nil) + if err == nil { + t.Fatalf("%q: accepted", raw) + } + if strings.Contains(err.Error(), "s3cret-pw") { + t.Fatalf("%q: error leaks password: %v", raw, err) + } + if !strings.Contains(err.Error(), "alice") && !strings.Contains(err.Error(), "xxxxx") { + t.Fatalf("%q: error lost the redacted URL entirely: %v", raw, err) + } + } +} + func TestGitEnv(t *testing.T) { sel, _ := Resolve(Config{URL: "http://p:3128", Hosts: []string{"github.ibm.com", "*.github.ibm.com"}, NoProxy: []string{"github.com"}}, nil) got := strings.Join(sel.GitEnv(), "\n") diff --git a/src/deploy/README.md b/src/deploy/README.md index c6705aa0d6..802b7bb622 100644 --- a/src/deploy/README.md +++ b/src/deploy/README.md @@ -20,6 +20,7 @@ Use it indirectly through the dashboard terminal link. If a terminal pane says i - `k8s/` — the Kustomize base (`kustomization.yaml`) with namespace, deployment, service, PVC, Secret, ConfigMap and RBAC manifests, plus the custom error backend (`error-pages.yaml`), the hub backup CronJob (`backup-cronjob.yaml`) and the node-prep DaemonSet (`node-prep/`), which are applied separately. The OpenShift Route lives in `kustomize/overlays/openshift/`. - `inference/` — sample in-cluster OpenAI-compatible inference deployment and RBAC. - `openshell.md` — cluster-side notes for the opt-in `sandbox.runtime: openshell` (ADR-0026): prerequisites, the upstream Helm install, the gateway settings and PVC labels hive's runtime depends on, and how to verify. No manifests are vendored. +- `substrate.md` — cluster-side notes for the opt-in `sandbox.runtime: substrate` (ADR-0026 phase 2): the Agent Substrate operator install, WorkerPool and SandboxConfig, the pod-certificate feature gates, the router client bundle hive needs, and how to verify. No manifests are vendored. - `docker-compose.architect.yaml`, `hive-quickstart.yaml`, `hive-level*.yaml`, `architect-only.yaml`, `hive.yaml` — example deployment/configuration manifests. - `blue-green-deploy.sh`, `bootstrap-lxc.sh`, `create-lxc.sh` — operational scripts for **Docker Compose** deployments (LXC bootstrap and a blue-green Compose upgrade). Not "non-Kubernetes" generally: none of the three shells out to `podman` or reads `HIVE_DEPLOY_RUNTIME`, so none of them applies to the Quadlet stack in `quadlet/` below. See [Deployment helper scripts](../docs/deployment-scripts.md) for each script's scope and where a Podman operator should go instead. - `quadlet/` — Podman Quadlet units for the standalone deployment (ADR-0017). `hive.container` and `hive-data.volume` start the Hive service under `systemd`, rootful or rootless; `hive.network` and `hive-gateway.container` put the authenticating gateway in front of it with the same published-port split as the Compose stack — 3001 published, 7681 never. See [the operator guide](../docs/podman-standalone-quadlet.md). Docker Compose remains the default runtime and is unaffected. diff --git a/src/deploy/entrypoint.sh b/src/deploy/entrypoint.sh index 650739e4e4..d5d0b464aa 100644 --- a/src/deploy/entrypoint.sh +++ b/src/deploy/entrypoint.sh @@ -64,7 +64,9 @@ EXIT_NET_ADMIN_REQUIRED=77 # file is the only copy of the live config, and mutating it at boot risks # losing owner customisations with no warning. The old name is picked up # read-only until the first save writes the new one, after which both -# exist and the new one wins. +# exist and the new one wins. Once the new name is the boot source, every +# clean boot refreshes the old name from it (#17862) so the fallback is +# never months stale; see hive_refresh_legacy_fallback. HIVE_CONFIG_PATH="${HIVE_CONFIG:-/etc/hive/hive.yaml}" HIVE_CONFIG_RUNTIME="/data/hive.yaml.runtime" HIVE_CONFIG_RUNTIME_LEGACY="/data/hive.yaml.bak" @@ -775,18 +777,89 @@ for _cfg in "$HIVE_CONFIG_RUNTIME" "$HIVE_CONFIG_RUNTIME_LEGACY" /data/hive.yaml done unset _cfg +# hive_file_mtime echoes a file's mtime as epoch seconds (0 when unreadable). +# Both stat dialects are tried because the image is Debian but the helper +# must also behave on the BusyBox/macOS shells the tests run under. +hive_file_mtime() { + stat -c %Y "$1" 2>/dev/null || stat -f %m "$1" 2>/dev/null || echo 0 +} + # hive_runtime_config_read echoes the path to read the persisted runtime # config from: the new name when it is present and non-empty, else the # legacy name when that is, else empty. Read-only — it never creates, # renames or removes anything on the PVC. +# +# #17862: falling back to the legacy name is never routine any more. Every +# clean boot refreshes it from the runtime config (hive_refresh_legacy_fallback), +# so reaching it means the runtime config was lost or truncated since the +# last boot. Say so LOUDLY on stderr with both files' ages: a hive that +# silently booted a 3-month-old hive.yaml.bak (level 3 instead of 6) is the +# incident this guards against. When the legacy copy is OLDER than the +# runtime file it is replacing — i.e. the runtime file was written after +# the last refresh and then damaged — the WARN says the fallback is stale +# and names the overlay/audit paths an operator can reconstruct from. hive_runtime_config_read() { if [ -f "$HIVE_CONFIG_RUNTIME" ] && [ -s "$HIVE_CONFIG_RUNTIME" ]; then echo "$HIVE_CONFIG_RUNTIME" elif [ -f "$HIVE_CONFIG_RUNTIME_LEGACY" ] && [ -s "$HIVE_CONFIG_RUNTIME_LEGACY" ]; then + _rt_state="missing" + _rt_mtime=0 + if [ -f "$HIVE_CONFIG_RUNTIME" ]; then + _rt_state="present but EMPTY (0 bytes — likely truncated by a kill mid-save)" + _rt_mtime="$(hive_file_mtime "$HIVE_CONFIG_RUNTIME")" + fi + _lg_mtime="$(hive_file_mtime "$HIVE_CONFIG_RUNTIME_LEGACY")" + _now="$(date +%s)" + _lg_age_h=$(( (_now - _lg_mtime) / 3600 )) + echo "[entrypoint] WARN: runtime config $HIVE_CONFIG_RUNTIME is $_rt_state; FALLING BACK to legacy $HIVE_CONFIG_RUNTIME_LEGACY (last refreshed ${_lg_age_h}h ago)." >&2 + if [ "$_rt_mtime" -gt "$_lg_mtime" ] 2>/dev/null; then + echo "[entrypoint] WARN: the legacy fallback is OLDER than the damaged runtime config ($(( (_rt_mtime - _lg_mtime) / 3600 ))h behind). Settings saved since then (acmm_level, agent roster, pauses, models) are NOT in this file — check /data/hive.yaml.dashboard and /data/audit.jsonl before trusting the booted config. See hivecommons/hive#17862." >&2 + fi + unset _rt_state _rt_mtime _lg_mtime _now _lg_age_h echo "$HIVE_CONFIG_RUNTIME_LEGACY" fi } +# hive_refresh_legacy_fallback copies a GOOD runtime config over the legacy +# fallback name so the fallback is at most one boot stale (#17862). Before +# this, hive.yaml.bak was frozen at the rename (Aug 2026) on every hive, and +# a truncated runtime file booted a months-old config. +# +# Only runs when THIS boot is reading from the new name (so the legacy file +# is strictly older, never the only copy — the Docker/LXC worry behind the +# original never-write rule) and the runtime file parses as YAML with a +# project section (so a half-written file is never promoted to fallback). +# Copies to a temp name and renames so the fallback is never itself +# truncated by a kill. Best-effort: a failure here only means the fallback +# stays at its previous age. +hive_refresh_legacy_fallback() { + [ "$1" = "$HIVE_CONFIG_RUNTIME" ] || return 0 + [ -s "$HIVE_CONFIG_RUNTIME" ] || return 0 + if command -v python3 >/dev/null 2>&1; then + python3 -c ' +import sys, yaml +with open(sys.argv[1]) as f: + d = yaml.safe_load(f) +sys.exit(0 if isinstance(d, dict) and isinstance(d.get("project"), dict) else 1) +' "$HIVE_CONFIG_RUNTIME" 2>/dev/null || { + echo "[entrypoint] WARN: $HIVE_CONFIG_RUNTIME does not parse as a hive config; legacy fallback $HIVE_CONFIG_RUNTIME_LEGACY left untouched" >&2 + return 0 + } + fi + if cmp -s "$HIVE_CONFIG_RUNTIME" "$HIVE_CONFIG_RUNTIME_LEGACY" 2>/dev/null; then + return 0 + fi + _lg_tmp="${HIVE_CONFIG_RUNTIME_LEGACY}.tmp.$$" + if cp "$HIVE_CONFIG_RUNTIME" "$_lg_tmp" 2>/dev/null && mv -f "$_lg_tmp" "$HIVE_CONFIG_RUNTIME_LEGACY" 2>/dev/null; then + hive_harden_runtime_config "$HIVE_CONFIG_RUNTIME_LEGACY" + echo "[entrypoint] Legacy fallback $HIVE_CONFIG_RUNTIME_LEGACY refreshed from $HIVE_CONFIG_RUNTIME" + else + rm -f "$_lg_tmp" 2>/dev/null + echo "[entrypoint] WARN: could not refresh legacy fallback $HIVE_CONFIG_RUNTIME_LEGACY from $HIVE_CONFIG_RUNTIME" >&2 + fi + unset _lg_tmp +} + # hive_ghe_git_host echoes this hive's configured GitHub host when it is NOT # public github.com (i.e. a GitHub Enterprise instance such as github.ibm.com), # and nothing at all otherwise. Derived the same way the Go binary's @@ -1022,6 +1095,7 @@ if [ "$IS_KUBERNETES" = "true" ]; then # result equal to its own input on every boot. HIVE_CONFIG_BOOT="$(hive_runtime_config_read)" if [ -n "$HIVE_CONFIG_BOOT" ]; then + hive_refresh_legacy_fallback "$HIVE_CONFIG_BOOT" # Steady state: boot from what this hive last had. The ConfigMap seed is # deliberately NOT merged — it is frozen at provision and every field it # could contribute is either already here or re-asserted by the hub on @@ -1211,6 +1285,7 @@ else # first boot on new code seeds hive.yaml.runtime from the legacy content # without touching the legacy file itself. HIVE_CONFIG_SOURCE="$(hive_runtime_config_read)" + hive_refresh_legacy_fallback "$HIVE_CONFIG_SOURCE" if [ -f "$HIVE_CONFIG_PATH" ] && [ -s "$HIVE_CONFIG_PATH" ] && [ -z "$HIVE_CONFIG_SOURCE" ]; then # First boot: config exists but no PVC runtime config yet — seed it cp "$HIVE_CONFIG_PATH" "$HIVE_CONFIG_RUNTIME" diff --git a/src/deploy/k8s/sandbox-substrate-preflight-rbac.yaml b/src/deploy/k8s/sandbox-substrate-preflight-rbac.yaml new file mode 100644 index 0000000000..7a60c377ee --- /dev/null +++ b/src/deploy/k8s/sandbox-substrate-preflight-rbac.yaml @@ -0,0 +1,72 @@ +# Lets the spoke's Substrate preflight (sandbox.runtime: substrate, #17698; +# crux/sandbox/substrate_preflight_cluster.go) read the cluster facts behind +# a failing kick: the operator's CRDs and workloads, WorkerPools and +# SandboxConfigs, RuntimeClasses, node kernels/KVM, and whether the +# certificates.k8s.io APIs behind the PodCertificateRequest and +# ClusterTrustBundle feature gates are served. Read-only, get/list only; +# without it the preflight emits one warning and skips these checks. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: hive-substrate-preflight + labels: + app.kubernetes.io/name: hive +rules: + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + resourceNames: ["workerpools.ate.dev", "sandboxconfigs.ate.dev"] + verbs: ["get"] + - apiGroups: ["ate.dev"] + resources: ["workerpools", "sandboxconfigs"] + verbs: ["get", "list"] + - apiGroups: ["node.k8s.io"] + resources: ["runtimeclasses"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: hive-substrate-preflight + labels: + app.kubernetes.io/name: hive +subjects: + - kind: ServiceAccount + name: hive + namespace: hive +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: hive-substrate-preflight +--- +# The operator namespace (ate-system by default; the preflight derives it +# from sandbox.substrate.endpoint's api..svc host). +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: hive-substrate-preflight + namespace: ate-system + labels: + app.kubernetes.io/name: hive +rules: + - apiGroups: ["apps"] + resources: ["deployments", "daemonsets"] + verbs: ["get", "list"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: hive-substrate-preflight + namespace: ate-system + labels: + app.kubernetes.io/name: hive +subjects: + - kind: ServiceAccount + name: hive + namespace: hive +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: hive-substrate-preflight diff --git a/src/deploy/openshell.md b/src/deploy/openshell.md index 7653c1ad9f..706461a648 100644 --- a/src/deploy/openshell.md +++ b/src/deploy/openshell.md @@ -16,7 +16,8 @@ record it alongside the hive config. | Requirement | Why | How to check | |---|---|---| -| Node kernel **≥ 6.2** with Landlock ABI ≥ 3 | OpenShell's sandbox qualification fails below it | `kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kernelVersion}'`. Of the clusters hive runs on, LKE (Debian 13, 6.12) qualifies; OKE UEK7 (5.15) and RHCOS 9.x (5.14) do not. | +| Node kernel **≥ 6.2** with Landlock **active** | OpenShell's sandbox qualification fails below it; a new-enough kernel built without Landlock in the LSM list fails the same way | `kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kernelVersion}'`, then from a privileged debug pod `cat /sys/kernel/security/lsm` must contain `landlock`. Of the clusters hive runs on, LKE (Debian 13, 6.12) and OKE on OL9/UEK8 (6.12) qualify; OKE UEK7 (5.15) and RHCOS 9.x (5.14) do not. | +| The **Agent Sandbox** controller (`sandboxes.agents.x-k8s.io`) | the gateway's Kubernetes driver creates `Sandbox` objects; the chart's preflight refuses to install without the API | `kubectl apply -f https://github.com/kubernetes-sigs/agent-sandbox/releases/latest/download/sandbox.yaml`, then `kubectl api-resources \| grep agents.x-k8s.io` | | A CNI that **enforces** egress `NetworkPolicy` (Calico, Cilium) | the gateway proxies egress; the pod must not be able to go around it. Flannel accepts policies and ignores them | create a deny-all policy in a scratch namespace and confirm a connection is actually refused, not just that the object exists | | Cluster-admin for the install | CRDs, the gateway's RBAC, PKI secrets | — | | A **ReadWriteMany** claim for hive's data when sandboxes may land on other nodes | the workspace is shared by mounting the same PVC | same caveat as the Job runtime | @@ -29,9 +30,17 @@ helm upgrade --install openshell \ --version \ --namespace openshell --create-namespace \ --set server.drivers.kubernetes.allowDriverConfig=true \ - --set-string 'server.env.OPENSHELL_TELEMETRY_ENABLED=false' + --set server.telemetryEnabled=false \ + --set server.workspaceStorageClass= \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.allowed_pod_labels={app.kubernetes.io/managed-by}' \ + --set 'gatewayConfig.openshell\.drivers\.kubernetes.allowed_pod_annotations={hive.dev/*}' ``` +`gatewayConfig` is rendered to TOML: its top-level keys are table names +written as dotted strings (`openshell.drivers.kubernetes:` in a values file, +escaped as above on the command line), not nested maps; nesting +`openshell: {drivers: …}` breaks the config and crash-loops the gateway. + Then, for the workspace hive will use (hive's `agent_sandbox.openshell.namespace`, default `default`), install the workspace chart with driver config allowed so the gateway may read PVC metadata for admission: @@ -61,12 +70,30 @@ Settings hive's runtime depends on, in gateway terms: `kubectl label pvc hive-data openshell.ai/sandbox-attachable=true openshell.ai/sandbox-attachable-workspace=default` on the spoke's namespace is enough for the defaults. -- **`OPENSHELL_TELEMETRY_ENABLED=false`** on the gateway. Hive's threat model +- **`allowed_pod_labels` / `allowed_pod_annotations`** under + `[openshell.drivers.kubernetes]`. The gateway rejects any template label or + annotation it was not told about; hive labels sandboxes + `app.kubernetes.io/managed-by=hive-openshell` and annotates them + `hive.dev/policy-profile` when a profile is set. +- **`server.workspaceStorageClass`**. The gateway provisions a 2Gi + `workspace-` PVC for every sandbox even though hive mounts its own + claim. On a block-storage default class that provision + attach added ~20 s + to every kick on LKE (27 s vs 9 s warm start, [#17643](https://github.com/hivecommons/hive/issues/17643)); + point it at an NFS/RWX or otherwise fast class. +- **Sandbox uid**. The gateway runs every sandbox as `runAsUser`/`fsGroup` + `10001` by default, overriding the image's `USER`. The kick workspace hive + creates on the shared claim must be writable by that uid: either set + `server.sandboxUid`/`server.sandboxGid` to the uid crux runs as, or make the + claim's workspace directories group-writable for 10001. +- **`server.telemetryEnabled=false`** on the gateway. Hive's threat model treats sandbox metadata as spoke-private; keep upstream telemetry off. -- **A bearer token** for hive. Create one per the upstream access-control page - and hand it to the hive pod as an environment variable (a Secret-backed - `env` entry on the hive Deployment); name that variable in - `agent_sandbox.openshell.token_env`. The token never goes in `hive.yaml`. +- **A bearer token** for hive. Upstream authenticates clients by mTLS, OIDC + JWT, or a trusted reverse proxy; hive's client sends a static bearer token, + so "token" here means a JWT from the OIDC issuer the gateway trusts (see the + upstream access-control page). Hand it to the hive pod as an environment + variable (a Secret-backed `env` entry on the hive Deployment); name that + variable in `agent_sandbox.openshell.token_env`. The token never goes in + `hive.yaml`. `allowUnauthenticatedUsers` is for throwaway clusters only. - **Digest-pinned images**: the agent's `sandbox.image` should be `registry/image@sha256:…`. The gateway's own supervisor and sandbox runtime images are pinned by the chart version. @@ -91,11 +118,19 @@ Settings hive's runtime depends on, in gateway terms: `api.github.com` must be **denied** and `git fetch` must succeed. Sandboxes are labelled `app.kubernetes.io/managed-by=hive-openshell` and are deleted when the kick ends; one that is still there after a finished kick is a bug. + Sandbox names are `hk--` (the gateway caps names at + 19 characters) and the claim is mounted as volume `hive-workspace` + (`workspace` is reserved by the gateway). +5. Agent images must ship a glibc `sh`: debian/ubuntu/fedora/EL-based images + and hive's own image work, but busybox/musl images (alpine) fail at + `spawn sandbox entrypoint process '/bin/sh': Permission denied`. ## Not covered Hive does not preflight any of the above ([#17644](https://github.com/hivecommons/hive/issues/17644)) and has no Podman-driver path for standalone hosts yet. The hub provisioner -does not install the gateway for hosted spokes; that is a follow-up once the -prototype numbers ([#17643](https://github.com/hivecommons/hive/issues/17643)) -are in. +does not install the gateway for hosted spokes; that is a follow-up. The +prototype numbers are on [#17643](https://github.com/hivecommons/hive/issues/17643): +on LKE with the settings above, cold start 10.0 s vs 6.0 s for the Job runtime +(1.67×), warm 9.1 s vs 3.8 s (2.4×), +38 Mi per sandbox, egress policy +enforced. diff --git a/src/deploy/substrate.md b/src/deploy/substrate.md new file mode 100644 index 0000000000..707db2e458 --- /dev/null +++ b/src/deploy/substrate.md @@ -0,0 +1,118 @@ +# Installing Agent Substrate for `sandbox.runtime: substrate` + +Cluster-side notes for the opt-in Substrate sandbox runtime +([ADR-0026 phase 2](../docs/adr/0026-sandbox-runtime-selection.md#implementation-notes-phase-2-agent-substrate), +epic [#17695](https://github.com/hivecommons/hive/issues/17695); operator view +in [sandbox-isolation.md](../docs/sandbox-isolation.md#running-sandboxed-kicks-in-a-substrate-actor)). +Hive installs nothing: Substrate is a cluster-admin install, once per cluster, +and hive only talks to its Control API and router. Upstream is the source of +truth for the install — [agent-substrate/substrate → docs/guides/operator-install.md](https://github.com/agent-substrate/substrate/blob/main/docs/guides/operator-install.md), +the [API guide](https://github.com/agent-substrate/substrate/blob/main/docs/reference/api-guide.md) +and [sandbox classes](https://github.com/agent-substrate/substrate/blob/main/docs/reference/sandbox-classes.md) — +and no manifests are vendored here; pin the release you install and record it +alongside the hive config. + +## Prerequisites to check before installing + +| Requirement | Why | How to check | +|---|---|---| +| Kubernetes with `ClusterTrustBundle`, `ClusterTrustBundleProjection` and `PodCertificateRequest` feature gates and `certificates.k8s.io/v1beta1` enabled | actor and client identities are pod certificates (SPIFFE); the podcert controller does not start without them. Still off by default as of Kubernetes 1.36 | `kubectl get --raw /metrics \| grep 'kubernetes_feature_enabled{name="PodCertificateRequest"'` and `kubectl api-resources --api-group=certificates.k8s.io`. Managed control planes (OKE, LKE) need the provider to enable them; OpenShift needs a `FeatureGate` CR | +| Standard Linux nodes with **cgroup v2** for the `gvisor` class | `ateworker-gvisor` runs `runsc` inside the worker pod; there is no kernel floor beyond ≥ 5.8 | `kubectl get nodes -o jsonpath='{.items[*].status.nodeInfo.kernelVersion}'`; `stat -fc %T /sys/fs/cgroup` on a node → `cgroup2fs` | +| `/dev/kvm` on nodes for the `microvm` class | Kata on Cloud Hypervisor guests; `atelet` advertises the `ate.dev/kvm` device only where it exists | `kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.status.allocatable.ate\.dev/kvm}{"\n"}{end}'` after install | +| Nodes trusted to run worker pods with `SYS_ADMIN` and `NET_ADMIN` | workers are `privileged: false` but keep those two for mounts and networking | on OpenShift, an SCC granting them to the `ate-system` worker ServiceAccount | +| A CNI that **enforces** egress `NetworkPolicy` | Substrate's `atenet` dataplane carries actor egress; the worker pod itself must not be a side door | create a deny-all policy in a scratch namespace and confirm a connection is refused | +| PostgreSQL and S3-compatible object storage | Control API state; actor snapshots (`snapshot_on_idle`) | `ate-setup` deploys bundled ones for a lab; use managed services in production | +| Cluster-admin for the install | CRDs (`workerpools`, `sandboxconfigs`, `egresspolicies`, …), ValidatingAdmissionPolicies, RBAC, PKI | — | + +### Which hive clusters qualify (survey 2026-11-11) + +| cluster | kernel / Kubernetes | `gvisor` | `microvm` | +|---|---|---|---| +| hive-oke | OL9, UEK8 6.12, k8s 1.34.10 | yes, once the podcert feature gates are enabled on the OKE control plane | needs KVM-capable shapes (bare metal or nested virtualisation) | +| lke648397, hive-ci | Debian 13, 6.12 | yes; confirm the feature gates — LKE exposes no control-plane flags | KVM to confirm per plan | +| vllm-d, a-ks-wec2 | RHCOS 9.x, 5.14 | possible: SCC for the worker capabilities + `FeatureGate` CR (TechPreviewNoUpgrade) | only on KVM-capable nodes | + +## Install + +Build or download `ate-setup` from a pinned release of +`github.com/agent-substrate/substrate`, write the install document, and deploy: + +```yaml +# substrate-install.yaml +apiVersion: install.ate.dev/v1alpha1 +kind: SubstrateInstall +namespace: ate-system +atenet: + dataplane: envoy +images: + repo: /substrate + tag: +``` + +```shell +ate-setup deploy ate-system --config substrate-install.yaml \ + --credential-provider='{"name":"k8s.io"}' \ + --ingress-allowed-spiffe-ids=spiffe://cluster.local/ns//sa/ +``` + +`--ingress-allowed-spiffe-ids` must name hive's own pod identity: that is what +lets hive open `CONNECT` tunnels through `atenet-router` to the bridge in each +actor. Unknown keys in the document are rejected, not ignored. + +Then create the pieces hive's runtime depends on, in Substrate terms: + +- **An atespace** for hive (`sandbox.substrate.atespace`, default `default`), + and a principal for hive's token (`token_env` / `token_file`) that may + create and delete `ActorTemplate`, `Actor` and `EgressPolicy` and list + workers in it. `kubectl ate` shows the exact RBAC verbs for the release. +- **A `WorkerPool`** with `spec.sandboxClasses[].name: gvisor` (or `microvm`) + resolving the default `SandboxConfig` (`gvisor-default`), labelled so that + `sandbox.substrate.worker_selector` matches it; hive only uses pools whose + workers report `ACTIVE` and advertise a sandbox class. Taint the node pool + `ate.dev/sandboxClass=:NoSchedule` to reserve it; worker pods + tolerate it. +- **Projected files in the hive pod**: a ServiceAccount token with audience + `api.ate-system.svc` (→ `token_file`), the Control API CA via + `clusterTrustBundle` with signer `servicedns.podcert.ate.dev/identity` + (→ `ca_file`), and a `podCertificate` with signer + `podidentity.podcert.ate.dev/identity` (→ `router_client_bundle`) plus the + router CA (→ `router_ca_file`). The authentication guide upstream has the + volume stanza. +- **A digest-pinned sandbox image** containing `crux` (for the + `substrate-bridge` entrypoint) and the agent CLI, set as + `sandbox.substrate.image`; tags are refused at boot and at kick. Publishing + it is tracked in [#17697](https://github.com/hivecommons/hive/issues/17697). +- **Egress**: hive writes its own `EgressPolicy` per actor (`github.com`, + `api.github.com`, `allow_hosts`). Do **not** also configure Substrate's + credential injection for GitHub — actors run credential-free and hive + brokers pushes after the kick (ADR-0026, #17695). + +## Verify + +```shell +kubectl -n ate-system get deploy,ds # ateapi, atenet-router, podcert controller, atelet ready +kubectl get crd | grep ate.dev # workerpools, sandboxconfigs, egresspolicies, … +kubectl get workerpool -A # at least one pool with ACTIVE workers +kubectl get sandboxconfig gvisor-default -o yaml | grep defaultVersion +kubectl get nodes -o custom-columns=NAME:.metadata.name,KVM:.status.allocatable.ate\.dev/kvm +kubectl ate create actor smoke -a --template && kubectl ate delete actor smoke -a +``` + +From the hive side, start with `runtime: substrate` and watch the boot log: the +preflight warns when the Control API is unreachable, has no matching `ACTIVE` +worker, or its workers report no sandbox class. Then run one kick and confirm +in the router log that the actor's SPIFFE id made exactly the allowed +connections and that a connect to any other host was refused — that denial is +the point of the runtime. Latency and image numbers go back into this page +from the prototype ([#17697](https://github.com/hivecommons/hive/issues/17697)); +the fuller prover (CRDs, operator, feature gates, KVM) is +[#17698](https://github.com/hivecommons/hive/issues/17698). + +## Upgrades + +Substrate is pre-1.0; follow its version-skew policy (N ↔ N+1 only) and pin +the release and the sandbox image by digest. A `SandboxConfig` upgrade adds a +`runsc` version and keeps the previous one so suspended actors still restore. +Hive's generated client (`crux/sandbox/substrateapi`) tracks the fields it +uses; bump it with `generate.sh` against the matching upstream tag when the +Control API changes. diff --git a/src/deploy/test_entrypoint_runtime_config.sh b/src/deploy/test_entrypoint_runtime_config.sh index 4f97399d2c..6e21a20570 100755 --- a/src/deploy/test_entrypoint_runtime_config.sh +++ b/src/deploy/test_entrypoint_runtime_config.sh @@ -2,10 +2,13 @@ # Tests the hive.yaml.bak -> hive.yaml.runtime migration in entrypoint.sh. # # The rename is copy-forward: readers prefer /data/hive.yaml.runtime and fall -# back to the legacy /data/hive.yaml.bak, and NOTHING renames or removes the -# legacy file on the PVC. ~51 live hives carry only the legacy name, and on -# Docker/LXC that file is the single copy of the live config — mutating it at -# boot could lose owner customisations with no warning. +# back to the legacy /data/hive.yaml.bak, and NOTHING removes the legacy file +# on the PVC. ~51 live hives carry only the legacy name, and on Docker/LXC +# that file is the single copy of the live config. +# +# #17862 added one deliberate write: when the NEW name is the boot source and +# parses as a config, the legacy file is refreshed from it (temp + mv) so the +# fallback is never months stale, and falling back to it logs a loud WARN. # # Rather than grepping the script, this extracts the real resolver function and # executes it against real files, so it fails if the logic regresses. @@ -38,9 +41,9 @@ echo "=== entrypoint runtime-config migration tests ===" # Extract the resolver verbatim from the entrypoint so this tests the shipped # code, not a copy that can drift away from it. -RESOLVER="$(sed -n '/^hive_runtime_config_read() {/,/^}/p' "$ENTRYPOINT")" -if [ -z "$RESOLVER" ]; then - echo " FAIL: could not extract hive_runtime_config_read from $ENTRYPOINT" +RESOLVER="$(sed -n '/^hive_file_mtime() {/,/^}/p; /^hive_runtime_config_read() {/,/^}/p' "$ENTRYPOINT")" +if ! grep -q 'hive_runtime_config_read() {' <<<"$RESOLVER" || ! grep -q 'hive_file_mtime() {' <<<"$RESOLVER"; then + echo " FAIL: could not extract hive_runtime_config_read / hive_file_mtime from $ENTRYPOINT" exit 1 fi @@ -63,7 +66,7 @@ resolve() { export HIVE_CONFIG_RUNTIME HIVE_CONFIG_RUNTIME_LEGACY local out - out="$(eval "$RESOLVER"; hive_runtime_config_read)" + out="$(eval "$RESOLVER"; hive_runtime_config_read 2>/dev/null)" case "$out" in "$HIVE_CONFIG_RUNTIME") echo "new" ;; @@ -95,17 +98,96 @@ check "empty new + good legacy -> legacy" "legacy" "$(resolve "EMPTY" "old: cfg" # 6. Both empty is the same as neither. check "empty new + empty legacy -> empty" "none" "$(resolve "EMPTY" "EMPTY")" -# 7. The legacy file must never be renamed or deleted. On Docker/LXC it is the -# only copy of the live config, so a destructive migration could lose owner -# customisations silently. -if grep -Eq '(mv|rm)[^\n]*HIVE_CONFIG_RUNTIME_LEGACY' "$ENTRYPOINT"; then - echo " FAIL: entrypoint moves or removes the legacy runtime config" - echo " the migration must be copy-forward only" +# 7. The legacy file must never be deleted, and the only thing allowed to +# overwrite it is hive_refresh_legacy_fallback (#17862), which copies a +# newer, validated runtime config over it. On Docker/LXC it can be the +# only copy of the live config, so nothing else may touch it. +if grep -Eq 'rm[^\n]*HIVE_CONFIG_RUNTIME_LEGACY' "$ENTRYPOINT"; then + echo " FAIL: entrypoint removes the legacy runtime config" FAIL=$((FAIL + 1)) else - echo " PASS: legacy runtime config is never moved or removed" + echo " PASS: legacy runtime config is never removed" PASS=$((PASS + 1)) fi +REFRESH="$(sed -n '/^hive_refresh_legacy_fallback() {/,/^}/p' "$ENTRYPOINT")" +mv_total="$(grep -cE 'mv[^\n]*HIVE_CONFIG_RUNTIME_LEGACY' "$ENTRYPOINT" || true)" +mv_in_refresh="$(grep -cE 'mv[^\n]*HIVE_CONFIG_RUNTIME_LEGACY' <<<"$REFRESH" || true)" +check "legacy runtime config is only overwritten inside hive_refresh_legacy_fallback" "$mv_total" "$mv_in_refresh" + +# ── #17862: the legacy fallback is refreshed on clean boots and WARNs loudly ── +# +# The incident: a kill mid-save left a 0-byte hive.yaml.runtime, the resolver +# fell back to a hive.yaml.bak frozen three months earlier, and the hive +# booted a stale ACMM level with no indication in the log. Two properties +# close it: (a) a clean boot from the new name refreshes the legacy copy so it +# is at most one boot old, and (b) a fallback is announced on stderr. + +echo +echo "=== #17862: legacy fallback freshness and WARN ===" + +# resolve_stderr — stderr the resolver emits. +resolve_stderr() { + local new_content="$1" legacy_content="$2" + local dir + dir="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '$dir'" RETURN + HIVE_CONFIG_RUNTIME="$dir/hive.yaml.runtime" + HIVE_CONFIG_RUNTIME_LEGACY="$dir/hive.yaml.bak" + [ -n "$new_content" ] && printf '%s' "$new_content" > "$HIVE_CONFIG_RUNTIME" + [ -n "$legacy_content" ] && printf '%s' "$legacy_content" > "$HIVE_CONFIG_RUNTIME_LEGACY" + [ "$new_content" = "EMPTY" ] && : > "$HIVE_CONFIG_RUNTIME" + export HIVE_CONFIG_RUNTIME HIVE_CONFIG_RUNTIME_LEGACY + (eval "$RESOLVER"; hive_runtime_config_read 2>&1 >/dev/null) || true +} + +# (b) Falling back is loud; the steady state is silent. +warn_out="$(resolve_stderr "EMPTY" "old: cfg")" +if grep -q 'WARN.*FALLING BACK to legacy' <<<"$warn_out" && grep -q '0 bytes' <<<"$warn_out"; then + echo " PASS: empty runtime + legacy fallback logs a loud WARN naming the truncation" + PASS=$((PASS + 1)) +else + echo " FAIL: fallback to legacy did not WARN (got: '$warn_out')" + FAIL=$((FAIL + 1)) +fi +check "steady-state boot from the new name is silent" "" "$(resolve_stderr "new: cfg" "old: cfg")" + +# (a) hive_refresh_legacy_fallback: the legacy copy is overwritten from a +# valid runtime file, never from an invalid one, and only when the new +# name is the boot source. +HARDEN_FN="$(sed -n '/^hive_harden_runtime_config() {/,/^}/p' "$ENTRYPOINT")" +refresh_case() { + # refresh_case ; prints legacy content after. + local src="$1" rt="$2" lg="$3" dir + dir="$(mktemp -d)" + # shellcheck disable=SC2064 + trap "rm -rf '$dir'" RETURN + HIVE_CONFIG_RUNTIME="$dir/hive.yaml.runtime" + HIVE_CONFIG_RUNTIME_LEGACY="$dir/hive.yaml.bak" + HIVE_RUNTIME_USER="$(id -un)"; HIVE_RUNTIME_GROUP="$(id -gn)" + export HIVE_CONFIG_RUNTIME HIVE_CONFIG_RUNTIME_LEGACY HIVE_RUNTIME_USER HIVE_RUNTIME_GROUP + printf '%s' "$rt" > "$HIVE_CONFIG_RUNTIME" + printf '%s' "$lg" > "$HIVE_CONFIG_RUNTIME_LEGACY" + local arg="$HIVE_CONFIG_RUNTIME" + [ "$src" = legacy ] && arg="$HIVE_CONFIG_RUNTIME_LEGACY" + (eval "$HARDEN_FN"; eval "$REFRESH"; hive_refresh_legacy_fallback "$arg" >/dev/null 2>&1) || true + cat "$HIVE_CONFIG_RUNTIME_LEGACY" +} +# Contents are single-line here so $(...) newline stripping cannot bite. +GOOD_RT='{project: {org: acme}, acmm_level: 6}' +STALE_LG='{acmm_level: 3}' +check "clean boot from new name refreshes the legacy fallback" \ + "$GOOD_RT" "$(refresh_case new "$GOOD_RT" "$STALE_LG")" +check "boot from the legacy name leaves it untouched" \ + "$STALE_LG" "$(refresh_case legacy "$GOOD_RT" "$STALE_LG")" +if command -v python3 >/dev/null 2>&1 && python3 -c 'import yaml' 2>/dev/null; then + check "unparsable runtime config never overwrites the fallback" \ + "$STALE_LG" "$(refresh_case new 'project: [unterminated' "$STALE_LG")" + check "runtime config without a project section never overwrites the fallback" \ + "$STALE_LG" "$(refresh_case new '{acmm_level: 0}' "$STALE_LG")" +else + hive_test_skip "python3+yaml unavailable — cannot exercise the parse guard in hive_refresh_legacy_fallback" +fi # 8. Every write of the config path to the PVC targets the NEW name. Counted, # not just grepped: there are two such writes (the K8s post-merge snapshot diff --git a/src/docs/adr/0026-sandbox-runtime-selection.md b/src/docs/adr/0026-sandbox-runtime-selection.md index 3a6e062fde..aae4f451f1 100644 --- a/src/docs/adr/0026-sandbox-runtime-selection.md +++ b/src/docs/adr/0026-sandbox-runtime-selection.md @@ -1,6 +1,6 @@ # ADR-0026: Sandbox runtime selection -Status: Accepted (phase 1 implemented, runtime opt-in) — #17642, hivecommons/hive#17646 +Status: Accepted (phase 1 OpenShell implemented, runtime opt-in; measured on LKE in #17643: cold 1.67× / warm 2.4× the Job runtime, egress enforced — #17642, hivecommons/hive#17646; phase 2 Agent Substrate implemented, runtime opt-in — #17695, hivecommons/hive#17741) ## Context @@ -218,6 +218,93 @@ runs on, read on 2026-11-10: LKE (`lke648397`, `hive-ci`) run 6.12 on Debian 13 with Calico and qualify; OKE UEK7 (5.15), and the RHCOS 9.x OpenShift clusters (5.14) do not. +## Implementation notes (phase 2: Agent Substrate) + +Phase 2 (#17695, hivecommons/hive#17741) adds a **fourth** runtime, +`sandbox.runtime: substrate`, behind the same `sandbox.Launcher` seam. It is +[Agent Substrate](https://github.com/agent-substrate/substrate) (CNCF Sandbox, +donated by Google, Apache-2.0): a Kubernetes-hosted *actor* runtime where an +`ActorTemplate` (image, command, env, limits, volumes) is instantiated as an +`Actor` through a gRPC Control API (`ate.control.v1`), scheduled on a +`WorkerPool` whose pods run the workload under gVisor (`runsc`, class +`gvisor`) or in a KVM microVM (Kata on Cloud Hypervisor, class `microvm`). All +egress leaves through `atenet-router` under an `EgressPolicy`, and every actor +holds a SPIFFE identity issued through Kubernetes pod certificates. It was not +in the 2026-10-25 comparison; the write-up that led here is the Substrate +vs. crux research note, and the operator view is in +[sandbox-isolation.md](../sandbox-isolation.md#running-sandboxed-kicks-in-a-substrate-actor) +and [deploy/substrate.md](../../deploy/substrate.md). Like OpenShell it is +opt-in and inert unless selected. + +### How it maps onto the contract + +| Contract item | Substrate | What phase 2 does | +|---|---|---| +| Kernel-enforced confinement | gVisor user-space kernel or a microVM; worker pods are `privileged: false` but keep `SYS_ADMIN`/`NET_ADMIN` | selects the class through `worker_selector`; no change to hive | +| Default-deny egress | `atenet-egress` `EgressPolicy` per actor; rules are TLS-passthrough or terminate per host:port, no per-method read-only mode | `SubstrateEgressPolicy`: `github.com` and `api.github.com` on 443 plus `sandbox.substrate.allow_hosts`, TLS passthrough; everything else (and all UDP) denied | +| Credentials never in the sandbox | zero-credential model: actors have a SPIFFE id, credentials are injected at the router | actors run credential-free: no `HTTPS_PROXY`/MITM broker inside the actor and no GitHub credential enters it; hive does every push and GitHub write after the kick; Substrate's injection is **not** used (see the amendment below) | +| Workspace in, commits out | `durableDir` volumes and object-storage snapshots; no exec/attach API | an in-actor HTTP **bridge** (`crux substrate-bridge`) reached through the router's mTLS `CONNECT` tunnel: push tar → `POST /run` → pull tar; the hive-side workspace stays authoritative | +| Cleanup and timeouts | `DeleteActor` / `SuspendActor` / `DeleteActorTemplate` | one template + actor per kick, both deleted on success, failure and timeout (fresh bounded context, `ExitCode -1`, 16 MiB cap, as `job`/`openshell`) | + +### Deviations from Substrate's reference model + +| Substrate assumes | Phase 2 does | +|---|---| +| long-lived actors and reusable templates | per-kick `ActorTemplate` + `Actor`, named `hive-kick--` | +| Go client `go.ate.dev/substrate` | **not imported**: it needs Go 1.27 (hive is on 1.26) and a very large module graph. A trimmed, wire-compatible proto subset is generated in-repo (`crux/sandbox/substrateapi`, field numbers pinned by tests); zero new Go modules, +578 KB on the `crux` binary | +| image by tag or digest | digest-pinned `sandbox.substrate.image` only, enforced at boot (`AgentSandboxGateWarnings`) and at kick | +| no `policy_profile` concept | passed to the actor as `HIVE_POLICY_PROFILE`; Substrate has no annotations | +| snapshot/resume as the normal lifecycle | `snapshot_on_idle: true` suspends (snapshot to object storage) instead of deleting; only the latest suspended actor per agent per hive process is retained (the previous one is deleted on the next suspend; actors from before a restart are not reclaimed); resume-on-next-kick is not implemented (#17697) | +| `api.github.com` read-only | TLS passthrough like `github.com`; the actor holds no token and hive does every write after the kick | + +### Amendment (2026-11-12, #17695): credential-free actors + +Decision by the hive owner on #17695, option A: + +- Substrate actors run **credential-free**. There is no `HTTPS_PROXY` or MITM + broker inside the actor and no GitHub credential enters it; the actor's + egress is governed only by Substrate's `EgressPolicy`. +- Hive brokers pushes **post-kick**: it performs git pushes and GitHub writes + itself, from the workspace the bridge returns. +- **Phase 3 (future)**: SVID-authenticated broker delegation over the atenet + mTLS channel, so hive can delegate scoped GitHub writes to the actor without + a credential entering it. It is gated on hosted clusters serving the podcert + feature gates (`PodCertificateRequest` / `ClusterTrustBundle`, + `certificates.k8s.io/v1beta1`) and is tracked as a child issue of #17695. + +This supersedes the earlier statement that hive's credential brokering is +unchanged for Substrate. The podman, `job` and `openshell` runtimes keep +their proxy model (ADR-0002). + +### Cluster prerequisites and which hive clusters qualify + +Substrate's real requirement is **not** a kernel floor: the gVisor class runs +`runsc` inside the worker pod on any standard Linux node with cgroup v2; the +microVM class needs `/dev/kvm` (nested virtualisation on cloud shapes). Both +need the operator (`ate-setup deploy ate-system`: Control API, `atelet` +DaemonSet, `atenet-router`, podcert controller, PostgreSQL, snapshot object +storage) and Kubernetes with the `ClusterTrustBundle`, +`ClusterTrustBundleProjection` and `PodCertificateRequest` feature gates — still +off by default as of 1.36 — because actor identities are pod certificates. +Survey on 2026-11-11: + +| cluster | kernel / Kubernetes | `gvisor` class | `microvm` class | +|---|---|---|---| +| hive-oke | OL9, UEK8 6.12, k8s 1.34.10 | yes once the feature gates are on (OKE control plane: needs a support request or a self-managed control plane) | needs KVM-capable (bare-metal or nested-virt) shapes | +| lke648397, hive-ci | Debian 13, 6.12 | yes; LKE exposes no control-plane flags, so the podcert gates are the blocker — check with `kubectl get --raw /metrics \| grep kubernetes_feature_enabled` | KVM to confirm per plan | +| vllm-d, a-ks-wec2 | RHCOS 9.x, 5.14 | possible: the kernel is fine, but the worker's `SYS_ADMIN`/`NET_ADMIN` needs an SCC and the gates a `FeatureGate` CR (TechPreviewNoUpgrade) | only on KVM-capable nodes | + +### When to choose it over OpenShell + +- Choose **OpenShell** when the cluster already runs a ≥ 6.2 kernel with an + enforcing CNI and you want per-binary, per-method egress policy and the + shared-PVC workspace with no image changes. +- Choose **Substrate** when you want a stronger boundary than Landlock + + seccomp (gVisor or a VM), snapshot/suspend of idle agents, or SPIFFE + identities for the sandboxed workload, and you can run its operator, + PostgreSQL and object storage. Its egress policy is coarser (host:port, no + read-only API mode) and the workspace is copied, not shared. + ## Consequences - Hive gains a sandbox boundary that is checked before the agent runs, without @@ -233,8 +320,12 @@ with Calico and qualify; OKE UEK7 (5.15), and the RHCOS 9.x OpenShift clusters hive's policy. - Telemetry must be disabled explicitly. A preflight check should refuse a gateway that reports it enabled. +- Substrate (phase 2) adds a second operator-side stack (Control API, + atelet, atenet-router, PostgreSQL, object storage) and a copied workspace; + the hive-side credential model is untouched. Its preflight (#17698) and + prototype (#17697) follow the OpenShell ones. - If the prototype fails a gate, this record is rewritten to decide "no change" with the measured reason, and the Podman and Job runtimes remain the only ones. -Refs #2804, #6311, #13806, #14092. +Refs #2804, #6311, #13806, #14092, #17642, #17695. diff --git a/src/docs/api-reference.md b/src/docs/api-reference.md index 2f964ac8e0..594fe1d9dc 100644 --- a/src/docs/api-reference.md +++ b/src/docs/api-reference.md @@ -40,17 +40,17 @@ summary and the `health_deep_read` diagnostic tool. | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)); also `hub_build` (the hub's short SHA) and, once a stdio `hive-admin-mcp` client has called `/api/version`, `/api/status` or `/api/status/summary` with its `hive-admin-mcp/+` User-Agent, `last_seen_stdio_build` and `last_seen_stdio_at` (RFC3339) so the dashboard can flag build skew ([#12318](https://github.com/hivecommons/hive/issues/12318)) | `pkg/dashboard/api.go:46` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:47` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:77` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1838` | -| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `verification`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET` and no hub relay key, signature mismatch, undecodable payload, or a hub-relayed delivery rejected for `relay_signature_invalid`, `relay_timestamp_stale`, `relay_delivery_replayed` or `relay_delivery_missing` — and never changes the overall verdict; `verification` (`github-signed`, `relay (hub-signed)`, `none`) says how deliveries are authenticated | `pkg/dashboard/server.go:1839` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1840` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1846` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); every your-turn issue and PR row carries the critical path (#13646): `waits_on[]` (`{kind,ref}`, kind one of `pr`, `issue`, `external`, `base-check` (ref = the check red on the base branch), `approval`, `merge`; ref `#N` in the same repository or `owner/repo#N`), `unblocks` (transitive count of your-turn items waiting on it, cycle-guarded), `may_conflict_with[]` (PR numbers of the same repository sharing a closing issue or a known changed file; never an edge), `action` (`merge`, `release-hold`, `rebase`, `rerun` or `approve`), `action_ref`, `root_cause` and `blocker` (the root cause it collapses under); `repos[].yourTurn` has `issues`, `prs` (your turn, `CLOSE` excluded), `root_issues`, `root_prs`, `root_cause_count`, `order[]` (`{kind,repo,number}` by unblocks then age), `groups[]` (`{blocker,blocker_title,dependents[]}`) and `main_red` (`{check,since,fix,fix_your_turn}`); the top-level `yourTurn` sums the counts; `api_budget_pacing` `{enabled, active, factor, effective_interval_s, burn_per_min, reason}` is the API budget pacing state (`governor.api_budget_pacing`), and `evalIntervalS` is the loop interval after pacing | `pkg/dashboard/server.go:1851` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers, with the hive-wide `yourTurn` root-cause counts the phone widget reads (#13646); honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1852` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1855` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1856` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1857` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1858` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1859` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1849` | +| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `verification`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET` and no hub relay key, signature mismatch, undecodable payload, or a hub-relayed delivery rejected for `relay_signature_invalid`, `relay_timestamp_stale`, `relay_delivery_replayed` or `relay_delivery_missing` — and never changes the overall verdict; `verification` (`github-signed`, `relay (hub-signed)`, `none`) says how deliveries are authenticated | `pkg/dashboard/server.go:1850` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1851` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1857` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); every your-turn issue and PR row carries the critical path (#13646): `waits_on[]` (`{kind,ref}`, kind one of `pr`, `issue`, `external`, `base-check` (ref = the check red on the base branch), `approval`, `merge`; ref `#N` in the same repository or `owner/repo#N`), `unblocks` (transitive count of your-turn items waiting on it, cycle-guarded), `may_conflict_with[]` (PR numbers of the same repository sharing a closing issue or a known changed file; never an edge), `action` (`merge`, `release-hold`, `rebase`, `rerun` or `approve`), `action_ref`, `root_cause` and `blocker` (the root cause it collapses under); `repos[].yourTurn` has `issues`, `prs` (your turn, `CLOSE` excluded), `root_issues`, `root_prs`, `root_cause_count`, `order[]` (`{kind,repo,number}` by unblocks then age), `groups[]` (`{blocker,blocker_title,dependents[]}`) and `main_red` (`{check,since,fix,fix_your_turn}`); the top-level `yourTurn` sums the counts; `api_budget_pacing` `{enabled, active, factor, effective_interval_s, burn_per_min, reason}` is the API budget pacing state (`governor.api_budget_pacing`), and `evalIntervalS` is the loop interval after pacing | `pkg/dashboard/server.go:1862` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers, with the hive-wide `yourTurn` root-cause counts the phone widget reads (#13646); honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1863` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1866` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1867` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1868` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1869` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1870` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | @@ -93,7 +93,7 @@ summary and the `health_deep_read` diagnostic tool. | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:48` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1933` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](branding.md)) | `pkg/dashboard/server.go:1944` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:91` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:92` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:93` | @@ -121,9 +121,9 @@ summary and the `health_deep_read` diagnostic tool. | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1860` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1861` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1862` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1871` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1872` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1873` | ## Configuration @@ -327,7 +327,7 @@ Read the result from `GET /api/kick/{agent}/status`, which returns `status` of ` | `POST` | `/api/repos/{owner}/{repo}/items/{number}/hold` | Owner or repo write/maintain/admin | Add or remove a dashboard hold chip for an issue or PR. Body `{"held": true}` adds the hive's exact-match canonical `hive-pause/` label; `{"held": false}` removes exactly the labels causing the hold (`hive-pause/` and/or generic hold labels) and never removes `hive/` provenance. Every toggle writes an audit-log entry; the #8851 upgrade migration writes `/data/hive-hold-migration-.json` with audit-backed, provenance-only, and ambiguous legacy `hive/` classifications. | `pkg/dashboard/api.go:367`, `pkg/dashboard/api_repo_hold.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/decision` | Owner or repo write/maintain/admin, signed-in GitHub user | Record an operator decision from the activity monitor's row menu / selection bar. Body `{"decision": "approve"|"direction"|"close-superseded"|"close-wontfix", "comment": "…", "type": "issue"|"pr", "superseded_by": "#458"}`. On an issue parked at 045, `approve`/`direction` act exactly as a maintainer's `/hive approve` / `/hive decision ` comment would (labels, `approved-direction`, stage-engine Unparked fact, reply comment naming the operator). On a PR parked at 085 (`type: "pr"`), `approve` posts the `/lgtm` and `/approve` commands as two comments attributed to the operator, records the approval against the head as the human answer to a `requires_human` / decision-requested verdict (the lanes and the stage engine honour it like a trusted merger's Approve review), lifts `needs-human`, clears the sentinel label and acknowledges the bundle's sentinel findings for that head, re-derives the stage and drops the tracker cache — the response adds `released` (`approval-recorded`, `needs-human`, `sentinel-label`, `sentinel-acknowledged`) and `stage`; the merge gate decides from there (no review as the operator, no merge) — and `direction` records the text as a human change request against the head — the `/hive decision` fix lane — on a hive-authored PR only (409 otherwise). The close variants post a comment naming the operator (and the superseding PR when `superseded_by` is given) and close the item as not planned. Never merges. Every decision writes an audit-log entry (`repo_item_decision_*`). | `pkg/dashboard/api.go:368`, `pkg/dashboard/api_repo_item_decision.go` | | `POST` / `GET` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated` | Same as `…/hold` (owner or repo write), signed-in GitHub user; `decisions.delegate.enabled` (403 when off) | *Let hive decide*: enqueue a delegated-decision job in which the hive decides the parked item itself as the owner's delegate under `pkg/delegatedecision.PolicyText` and records it as a human decision (`/hive decision` comment with the `` marker, park labels off, `triage/accepted`, queue approval on a PR head). Body `{"type": "issue"|"pr", "reason", "caption", "sentinel_reason"}`. `POST` answers 202 with the job (`{id, status: "running"|"done"|"failed", decision: {verdict, rule, text, rationale, actions, keeps_park}, notes, error}`); 409 with the running job while one is in flight; `GET` reads it, from memory while this process holds the job and otherwise from the operator-action ledger (`from_ledger: true`; status, requester, start, verdict and rule, with the outcome's words in `notes` or `error`), so the last result outlives the 30-minute in-memory eviction and a restart. A job still running when the hive restarts is never resumed: the next process marks its ledger record `failed` (*interrupted by a hive restart before it finished, and not resumed …*) with a `repo_item_decision_delegated_failed` audit entry, and the operator asks again. Rails: never merges, refuses items wearing `hold`, never grants secrets/permissions (`needs-owner` + `hold` instead), recommends only on hive policy/governance questions (keeps 085). Audited as `repo_item_decision_delegated[_failed]`. See [dashboard-sections.md](dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:329`, `pkg/dashboard/api_repo_item_decide_delegated.go` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:592`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:594`, `pkg/dashboard/api_repo_item_accept_triage.go` | | `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:383`, `pkg/dashboard/api_acmm_eval.go` | | `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:384` | | `GET` | `/api/acmm-recommendation` | Dashboard auth/session | Advisory level-up recommendation (`acmmadvisor.Recommendation`, JSON): never changes the applied level — see [ACMM advisor](acmm-advisor.md) | `pkg/dashboard/api.go:385` | @@ -663,7 +663,7 @@ always resolved server-side from the validated token. | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:389` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:390` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:391` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1867` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1878` | ## Hub SaaS 102 routes — see [api-reference/hub-saas.md](api-reference/hub-saas.md). @@ -687,34 +687,34 @@ always resolved server-side from the validated token. | `GET` | `/api/hub/clusters` | Hub auth | List Clusters | `pkg/hub/saas.go:531` | | `GET` | `/api/hub/image-pulls` | Hub auth | Per-Release Image Pull Series | `pkg/hub/saas.go:377` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:522` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1807` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1808` | -| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1739` | -| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1740` | -| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1741` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1742` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1743` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1754` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1755` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1756` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1757` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1813` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1814` | +| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1745` | +| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1746` | +| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1747` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat | `pkg/hub/server.go:1748` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1749` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1760` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1761` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1762` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1763` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1765` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1775` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1776` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1777` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1778` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1779` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1780` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1781` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1782` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1785` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1788` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1789` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1796` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1797` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1798` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1815` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1820` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1822` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1771` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1781` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1782` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1783` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1784` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1785` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1786` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1787` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1788` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1791` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1794` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1795` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1802` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1803` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1804` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1821` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1826` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1828` | diff --git a/src/docs/api-reference/auth.md b/src/docs/api-reference/auth.md index 04d29177b1..e5e2eb8e3e 100644 --- a/src/docs/api-reference/auth.md +++ b/src/docs/api-reference/auth.md @@ -24,6 +24,6 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/openrouter/models` | Dashboard auth/session | Open Router Models | `pkg/dashboard/openrouter.go:47` | | `GET` | `/api/openrouter/credit` | Dashboard auth/session | Open Router Credit | `pkg/dashboard/openrouter.go:48` | | `GET` | `/openrouter/callback` | Public | Open Router Callback | `pkg/dashboard/openrouter.go:49` | -| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1860` | -| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1861` | -| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1862` | +| `POST` | `/api/github-app/recheck` | Dashboard auth/session | GitHub App Recheck | `pkg/dashboard/server.go:1871` | +| `POST` | `/api/github-app/install-clicked` | Dashboard auth/session | GitHub App Install Clicked | `pkg/dashboard/server.go:1872` | +| `GET` | `/gh-setup` | Public | GitHub App Setup Callback | `pkg/dashboard/server.go:1873` | diff --git a/src/docs/api-reference/dashboard-misc.md b/src/docs/api-reference/dashboard-misc.md index 12c2d99339..8ea3fbddee 100644 --- a/src/docs/api-reference/dashboard-misc.md +++ b/src/docs/api-reference/dashboard-misc.md @@ -84,7 +84,7 @@ The monitor renders CI and Review concurrently on one shared PR row; see | `POST` | `/api/hives/{id}/heartbeat` | Dashboard auth/session | Hives Heartbeat | `pkg/dashboard/api_contribute.go:389` | | `DELETE` | `/api/hives/{id}` | Owner only | Hives Delete | `pkg/dashboard/api_contribute.go:390` | | `POST` | `/api/hives/onboard` | Dashboard auth/session | Hives Onboard | `pkg/dashboard/api_contribute.go:391` | -| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1867` | +| `GET` | `/sso` | Public | SSO | `pkg/dashboard/server.go:1878` | ### Activity observation provenance diff --git a/src/docs/api-reference/health-status-events.md b/src/docs/api-reference/health-status-events.md index d3899ff250..c49f7afa4f 100644 --- a/src/docs/api-reference/health-status-events.md +++ b/src/docs/api-reference/health-status-events.md @@ -24,17 +24,17 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/version` | Dashboard auth/session | Build/version metadata; includes `upgradeMarker` (`target`, `current`, `attempts`, `maxAttempts`, `failed`, `requestedAt`, `lastError`) while a self-upgrade is in flight or has failed ([#6765](https://github.com/hivecommons/hive/issues/6765)), the hub-delivered `upgradePolicy` (`hub_managed`, `spoke_managed`, `schedule`, `schedule_hour`, `schedule_timezone`, `schedule_weekday`, `paused`, `branch`, `channel`, `target_sha`, `target_resolved`, `armed_target`, and `next_update_at` — the hub's RFC3339 ETA for the next promotion into the `stable` channel, omitted when unknown ([#10256](https://github.com/hivecommons/hive/issues/10256))) when available, and an `autoUpdate` object (`enabled`, `state` — one of `disabled`/`up_to_date`/`behind`/`retrying`/`failed`/`unknown` — `healthy`, `period`, `targetBranch`, `targetCommit`, `currentCommit`, `commitsBehind`, `lastAttemptAt`, `lastError`, `detail`, and `nextUpdateAt` — the hub's RFC3339 ETA for the next update, omitted when unknown, disabled or paused ([#10257](https://github.com/hivecommons/hive/issues/10257))) that never reports a failed or unknown update as healthy ([#6962](https://github.com/hivecommons/hive/issues/6962), [#6963](https://github.com/hivecommons/hive/issues/6963)). Also `startedAt` (RFC3339 process start — changes on every pod roll even when the build does not, so the dashboard can say "Restarted" vs "Upgraded") and, until the first status is published, `startupPhase`/`startupPhaseLabel` (the boot phase a fresh pod is in) ([#15788](https://github.com/hivecommons/hive/issues/15788)) | `pkg/dashboard/api.go:46` | | `GET` | `/api/version/release-notes` | Dashboard auth/session | Changelog gap between two revisions: `?from=&to=` returns `{from:{sha,version?}, to:{sha,ref,version?}, sections:[{version,date,categories:{added,changed,fixed,security,deprecated}}], unreleased?, truncated, source, error?}` — the `CHANGELOG.md` release sections present at `to` and absent at `from`, newest first, plus (when `to` is not a release tag) the `changelog.d/` fragments new at `to` under an `Unreleased (in this build)` section. Fetch failures return `source: "unavailable"` with `error` rather than an error status; results are cached per (from, to) for an hour ([#11065](https://github.com/hivecommons/hive/issues/11065)) | `pkg/dashboard/api.go:47` | | `POST` | `/api/release-channel` | Owner only | Hosted spoke self-service release-channel selector; relays `stable`/`candidate`/`edge` to the hub's existing switch-branch endpoint with the spoke dashboard-token proof, and reports the requested channel as pending until the Deployment image lands | `pkg/dashboard/api.go:77` | -| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1838` | -| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `verification`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET` and no hub relay key, signature mismatch, undecodable payload, or a hub-relayed delivery rejected for `relay_signature_invalid`, `relay_timestamp_stale`, `relay_delivery_replayed` or `relay_delivery_missing` — and never changes the overall verdict; `verification` (`github-signed`, `relay (hub-signed)`, `none`) says how deliveries are authenticated | `pkg/dashboard/server.go:1839` | -| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1840` | -| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1846` | -| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `code_scanning` `{enabled, repos[], notices[]}` is the code scanning sweep's picture, absent while the sweep is off everywhere — per repo `{repo, security_url, checked_at, open_alerts, by_severity{critical,high,medium,low,note}, qualifying, below_threshold, excluded, tracked, tracked_closed, untracked, open_issues, filed, updated, closed, truncated, notice, notice_kind, error}`, where `tracked` is open alerts listed on an open hive issue and `notice_kind` is `missing_permission` (the App lacks Code scanning alerts: Read) or `not_enabled` (see `docs/code-scanning.md`); `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot; `ghRateLimits.agent_caps` `{fleet_cap, fleet_cap_set, window_s, agents: [{agent, cap, source: agent|base_agent|fleet, base_agent, calls}]}` is the GitHub API budget panel's per-agent proxy cap (0 = no cap) and the calls the proxy counted against it in the current sliding hour (`calls` is null for an uncapped agent, which the proxy does not count); `api_budget_pacing` `{enabled, active, factor, effective_interval_s, burn_per_min, reason}` is the API budget pacing state (`governor.api_budget_pacing`), and `evalIntervalS` is the loop interval after pacing | `pkg/dashboard/server.go:1851` | -| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1852` | -| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1855` | -| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1856` | -| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1857` | -| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1858` | -| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1859` | +| `GET` | `/api/health` | Public | Basic health probe | `pkg/dashboard/server.go:1849` | +| `GET` | `/api/health/deep` | Public | Deep health probe; the `webhooks` check (`status`, `detail`, `mode`, `mode_reason_code`, `verification`, `rejected_1h`) warns when deliveries arrive but cannot be used — no `GITHUB_WEBHOOK_SECRET` and no hub relay key, signature mismatch, undecodable payload, or a hub-relayed delivery rejected for `relay_signature_invalid`, `relay_timestamp_stale`, `relay_delivery_replayed` or `relay_delivery_missing` — and never changes the overall verdict; `verification` (`github-signed`, `relay (hub-signed)`, `none`) says how deliveries are authenticated | `pkg/dashboard/server.go:1850` | +| `GET` | `/api/livez` | Public | Kubernetes liveness probe | `pkg/dashboard/server.go:1851` | +| `GET` | `/metrics` | Registered only when `HIVE_METRICS_ENABLED`; requires `Authorization: ****** (403 if the token is unset) | Prometheus metrics | `pkg/dashboard/server.go:1857` | +| `GET` | `/api/status` | Dashboard auth/session | Dashboard aggregate status; `labelPresets[]` (`{key,label,description,labels[],states[]}`) lists the built-in label-filter presets for the Projects cards, led by `needs-human` whose `labels[]` are the effective hard-suppress labels plus `needs-signal`, the hold labels, `stage/071-needs-author` and the reporter-trust awaiting label (#12172); accepts `?fields=a,b` for top-level selection and `?omit=repos` to drop heavy sections, and honors `Accept-Encoding: gzip`; `agents[]` includes busy-pane visibility fields (`kicksUndeliverable`, `busySince`, `lastTranscriptActivity`, `condition`, `conditionMessage`) when a Working pane cannot accept kicks; a working agent also carries `agents[].run` — the live per-run line (#14576): `{since, elapsedS, costUSD, tokens, priced, tests, filesChanged}`, the estimated spend/tokens since the kick was delivered (`lastKickAt`, falling back to `busySince`) (delta of the token collector's per-agent cumulative figure; `priced:false` when the agent has none yet), the last visible test-runner result (`pass`/`fail`/empty) and the last `N files changed` git summary (`-1` when none) parsed from the captured pane; top-level `fleetRun` `{running, hourUSD, hourPriced, dayUSD, weekUSD, monthUSD, windowsPriced}` is the governor strip's rotating `running N · $X this hour / today / this week / this month` figure (#15614) — the hour is the rise since the top of the clock hour, the wider windows are the daily cost ledger summed from 00:00 UTC today, Monday 00:00 UTC (ISO week) and the 1st 00:00 UTC plus the rise since the ledger's last snapshot, `windowsPriced:false` when there is no ledger yet; `sla` (fleet) and `repos[].sla` (per repo) carry the Delivery SLA block derived from the lifecycle timeline with no extra GitHub calls: `windows.{7d,30d}.{open_to_triage,triage_to_start,start_to_pr,pr_to_merge,open_to_merge}` as `{p50_h,p95_h,n}` (hours, nearest-rank), `history[]` of up to 30 daily buckets `{day,start_to_pr_p50,pr_to_merge_p50,open_to_merge_p50,n_started,n_pr,n_merged}` persisted under `/data/sla-history.json`, `stale_in_progress[]` (`{number,repo,started_at,age_h,agent}` — issues carrying `hive/in-progress` longer than the agent claim TTL with no linked PR), `stale_count`, `merged_7d`, `claim_ttl_s` and, on the fleet block, the effective `targets` (`start_to_pr_h`, `pr_to_merge_h`, `open_to_merge_h`); `staleSince` and `repos[].countsStale` / `countsStaleSince` mark totals served from last-good figures — the previous enumeration (#13382) or the ForgeState cache's last-good counts (#13376); `ci_failure_signals` `{count, last, kicks, unmatched}` is the webhook CI-failure signal store — distinct red (repo, head, check) signals retained, the most recent one `{repo, number, head_sha, check, conclusion, details_url, agent, at}`, webhook-triggered kicks/nudges since boot, and failure deliveries whose head matched no open PR; `workflow_approvals` `{approved, denied, permission_errors, last}` counts the hive's approvals of GitHub-gated workflow runs since boot (#16603) — runs approved, runs refused by the trust policy (once per head), `403`/`404` refusals for want of `actions: write`, and the most recent decision `{repo, number, head, run_id, outcome, at}`; `code_scanning` `{enabled, repos[], notices[]}` is the code scanning sweep's picture, absent while the sweep is off everywhere — per repo `{repo, security_url, checked_at, open_alerts, by_severity{critical,high,medium,low,note}, qualifying, below_threshold, excluded, tracked, tracked_closed, untracked, open_issues, filed, updated, closed, truncated, notice, notice_kind, error}`, where `tracked` is open alerts listed on an open hive issue and `notice_kind` is `missing_permission` (the App lacks Code scanning alerts: Read) or `not_enabled` (see `docs/code-scanning.md`); `hotspots` `{count, actionable, filed}` is the merge-hotspot advisor's one-line summary (files at/above the thresholds, those with a split recommendation, those with a modularize issue filed), absent when the advisor is not wired — details on `GET /api/hotspots`; `writeback.dedupe` `{enabled, dedupe_same_item, dedupe_burst, rate_capped, hourly_cap, burst_items, burst_window_s, same_item_ttl_s}` is the write-back gate's content layer — the effective `writeback.dedupe.*` settings and the last-hour counts of writes held as `duplicate`, `burst` and `rate-capped` (each also listed under `GET /api/writeback/suppressed`); `config.gate_regressions[]` `{gate, previous, effective, source}` is the boot-time safety-gate regression list (#16438) — gates a previous runtime overlay on the PVC carried that the running config lacks or loosens (`review.require_labels`, `project.issue_filter.require_labels`, `project.paused_repos[]`, `writeback.per_repo[].mode`), empty on a healthy boot; `ghRateLimits.agent_caps` `{fleet_cap, fleet_cap_set, window_s, agents: [{agent, cap, source: agent|base_agent|fleet, base_agent, calls}]}` is the GitHub API budget panel's per-agent proxy cap (0 = no cap) and the calls the proxy counted against it in the current sliding hour (`calls` is null for an uncapped agent, which the proxy does not count); `api_budget_pacing` `{enabled, active, factor, effective_interval_s, burn_per_min, reason}` is the API budget pacing state (`governor.api_budget_pacing`), and `evalIntervalS` is the loop interval after pacing | `pkg/dashboard/server.go:1862` | +| `GET` | `/api/status/summary` | Dashboard auth/session | Compact agent/governor/budget status summary for pollers; honors `Accept-Encoding: gzip` | `pkg/dashboard/server.go:1863` | +| `GET` | `/api/overview/issues.csv` | Dashboard auth/session | Cached Overview issue-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1866` | +| `GET` | `/api/overview/issues.json` | Dashboard auth/session | Cached Overview issue-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1867` | +| `GET` | `/api/overview/prs.csv` | Dashboard auth/session | Cached Overview PR-band export as browser-compatible CSV; filters `band`, `repo`/`repos`, `stale`, and `held` are ANDed. | `pkg/dashboard/server.go:1868` | +| `GET` | `/api/overview/prs.json` | Dashboard auth/session | Cached Overview PR-band export as typed JSON with `bands[]` metadata and row fields matching the CSV columns. | `pkg/dashboard/server.go:1869` | +| `GET` | `/api/events` | Dashboard auth/session | Server-sent event stream | `pkg/dashboard/server.go:1870` | | `GET` | `/api/swarm` | Dashboard auth/session | Current swarm status: display name, duration, active repo, expiry, persisted prep metrics when available, and idle-unlock fields. | `pkg/dashboard/swarm.go:310` | | `POST` | `/api/swarm` | Owner only | Start a 24h repo swarm for a configured repo, returning 409 when another swarm is active and 423 when a previous swarm keeps the next swarm locked until enough agents are idle unless body `{"repo":"owner/name","force":true}` is used; saves the active record before repo prep, persists prep metrics, and announces the start to Discord when configured. | `pkg/dashboard/swarm.go:311` | | `DELETE` | `/api/swarm` | Owner only | End the active swarm, score issues closed and PRs merged in the window, append it to history, and announce the result to Discord when configured. | `pkg/dashboard/swarm.go:312` | diff --git a/src/docs/api-reference/hub-server.md b/src/docs/api-reference/hub-server.md index 501f650d5d..8b88123277 100644 --- a/src/docs/api-reference/hub-server.md +++ b/src/docs/api-reference/hub-server.md @@ -35,37 +35,37 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/benchmarks/ingest` | Per-hive heartbeat bearer | Spoke push target for anonymized [benchmark records](../benchmarks.md): body `{hive_id, records[]}`; the hub recomputes the hive hash from the authenticated id, rejects records whose `source.hive_id` differs (`401` bad bearer, `403` unregistered hive, `400` malformed), merges accepted records by id into `/data/benchmarks//.json` and returns `202 {accepted, rejected, files}` | `pkg/hub/benchmarks.go` | | `GET` | `/api/hub/benchmarks` | Public | Benchmark publisher status: `last_ingest_at`, `last_publish_at`, `last_publish_files`, `last_error`, `pending_files`, `hive_count`, `record_count` and per-hive-hash `hives{}` counts; the hub commits pending month files hourly to `hivecommons/benchmarks` (`HIVE_BENCHMARKS_REPO`) via its App identity | `pkg/hub/benchmarks.go` | | `GET` | `/api/reach` | Hub admin | PR Reach Report (?pr=NNN or ?recent=K) | `pkg/hub/saas.go:522` | -| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1807` | -| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1808` | -| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1739` | -| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1740` | -| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1741` | -| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1742` | -| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1743` | -| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1754` | -| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1755` | -| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1756` | -| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1757` | +| `GET` | `/fleet` | Hub handler-specific | My-Hives Fleet Page (static; data via `/api/saas/my-hives`) | `pkg/hub/server.go:1813` | +| `GET` | `/my-hives` | Hub handler-specific | 301 redirect to `/fleet` (query preserved) | `pkg/hub/server.go:1814` | +| `POST` | `/api/shared-agent-pool/leases` | Hub handler-specific | Acquire or renew a shared CLI session lease | `pkg/hub/server.go:1745` | +| `PUT` | `/api/shared-agent-pool/leases` | Hub handler-specific | Renew an active shared CLI session lease | `pkg/hub/server.go:1746` | +| `DELETE` | `/api/shared-agent-pool/leases` | Hub handler-specific | Release a shared CLI session lease | `pkg/hub/server.go:1747` | +| `POST` | `/api/heartbeat` | Hub handler-specific | Heartbeat. A request carrying `X-Hive-Activity-Update` is an activity-only update ([#15963](https://github.com/hivecommons/hive/issues/15963)): body `{"activity_update":{"hive_id","activity"}}`, the same per-hive bearer, accepted only for a hive already in the registry (`404` otherwise); it replaces the in-memory activity digest and nothing else, and the signed response carries `ok`, `activity_etag` and `activity_events_ack` (the highest activity event sequence number the hub holds for the hive, [#15964](https://github.com/hivecommons/hive/issues/15964); full heartbeat responses carry it too) | `pkg/hub/server.go:1748` | +| `POST` | `/api/task-status` | Hub handler-specific | Task Status | `pkg/hub/server.go:1749` | +| `GET` | `/api/registry` | Hub handler-specific | Registry | `pkg/hub/server.go:1760` | +| `GET` | `/api/hub/leaderboard` | Hub handler-specific | Leaderboard | `pkg/hub/server.go:1761` | +| `GET` | `/api/hub/stats` | Hub handler-specific | Stats | `pkg/hub/server.go:1762` | +| `GET` | `/api/fleet-stats` | Hub handler-specific | Fleet Stats | `pkg/hub/server.go:1763` | | `GET` | `/api/hub/evidence` | Hub admin | Read-only cross-hive evidence drill-down from each spoke's stored heartbeat `evidence` summary ([#12116](https://github.com/hivecommons/hive/issues/12116)); non-admins get `403`. Returns `spokes`, `reporting`, pooled `calibration` (bucket 0–5 → `{n, bad, rate, lo, hi}`; the hub view folds the six buckets into four reviewer-confidence ranges), `findings[]` (the plain-language *What this says* sentences, generated from the same fields; [#12456](https://github.com/hivecommons/hive/issues/12456)), `by_acmm_level` (`level`, `hives`, `merged_90d`, `revert_rate`, and `time_to_merge` — `median_of_hive_medians_s`, `hives`, `n` — the median of the level's per-hive median time-to-merge, omitted when no hive at the level reports one; [#12464](https://github.com/hivecommons/hive/issues/12464)), `family_false_approve_rate` (`same`/`cross`/`mixed`, hives classed by their reviewer vs author family sets), fleet `record_coverage` (merged PRs with an evidence record over `prs_merged_90d`), fleet panels ([#12370](https://github.com/hivecommons/hive/issues/12370)) `outcomes_by_author_kind[]` (`kind`, `hives`, `n`, `merged`, `reverted`, `merge_rate`, `revert_rate`), `write_counters[]` (`tier`, `hives`, `attempted`, `blocked`, `block_rate`) and `shadow_gate` (`review_gate` / `intent` → `held` / `passed` outcomes with `revert_rate`), each omitted while no spoke reports it and repeated per `by_acmm_level[]` row, and per hive `hives[]` with `acmm_level`, `prs_merged_90d`, `record_coverage`, `family_mix`, false-approve / false-block / revert rates, families, a calibration series the raw `counts` behind them (including `by_author_kind`) so the drill-down can re-pool a hive / ACMM-level selection client-side, and `criteria[]`: the per-axis criteria table derived hub-side from the quadrant criteria (`axis`, `label`, `scored`, `criteria[]` with `name`, `label`, `status` `met`/`unmet`/`not_reported`, `display`/`score` when measured, `reason` otherwise; offline, evidence-less or unscored hives are `not_reported`, never `unmet`; [#12463](https://github.com/hivecommons/hive/issues/12463)). Every rate is `{n, rate, lo, hi}` with a 95% Wilson interval; `rate`/`lo`/`hi` are null when `n` < 10. Window selection ([RFC #12062](https://github.com/hivecommons/hive/issues/12062)): `?hours=24|168|720` pools each hive's matching heartbeat `windows` block instead of its 90-day block (any other or absent `hours` → 90 days); `?from=YYYY-MM-DD|RFC3339[&to=…]` (date-only `to` is inclusive, `to` defaults to now, `to` ≥ `from`, span ≤ 366 days, else `400`) is served from the smallest reported window that reaches back to `from` and flagged `window.approximate`. The response carries `window` (`hours`, `key`, `label`, `from`, `to`, `approximate`, `coverage_available` — `record_coverage` is only computed for the 90-day window because `prs_merged_90d` is a 90-day count), `window_unreported` and per-hive `window_unreported` for hives whose spoke predates per-window reporting. `?hive=[,…]` (repeatable) and `?level=` restrict the registry server-side to those hives / that ACMM level before pooling. Hub admins open it from the Fleet quadrant card or its **🔬 Evidence** button ([#12117](https://github.com/hivecommons/hive/issues/12117)), pick a window (24h / 7d / 30d / 90d / custom), filter it by hive and ACMM level and read each section's ⓘ explainer; the per-hive author-kind revert counts behind the *Revert rate by author kind* panel are `hives[].counts.by_author_kind` ([#12326](https://github.com/hivecommons/hive/issues/12326)). Never consulted by merge decisions | `pkg/hub/evidence.go` | | `GET` | `/api/hub/evidence/report` | Hub admin | Plain-language report of the hub evidence view ([#12456](https://github.com/hivecommons/hive/issues/12456)), rendered server-side from the same `EvidenceView` as `/api/hub/evidence` so every figure reconciles with the modal; non-admins get `403`. Same `hours=` / `from=` / `to=` window and `hive=` / `level=` filter parameters. `format=md` (default, `text/markdown`) or `format=html` (standalone page, inline CSS, no scripts or external assets, hive names escaped); any other value → `400`. `download=1` sets `Content-Disposition: attachment; filename="hive-evidence---.md|html"`, otherwise inline. Sections: Summary (the generated findings), What we measured and why, The numbers, How to read these, What is not reported yet and why, Grade per section, footer link to `review-evidence.md`. `Cache-Control: no-store`. Read-only; never consulted by merge decisions | `pkg/hub/evidence_report.go` | -| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1765` | -| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1775` | -| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1776` | -| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1777` | -| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1778` | -| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1779` | -| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1780` | -| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1781` | -| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1782` | -| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1785` | -| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1788` | -| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1789` | -| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1796` | -| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1797` | -| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1798` | -| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1815` | -| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1820` | -| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1822` | +| `GET` | `/api/hub/version` | Hub handler-specific | Hub Version | `pkg/hub/server.go:1771` | +| `DELETE` | `/api/hub/registry/{id}` | Hub handler-specific | Registry Delete | `pkg/hub/server.go:1781` | +| `POST` | `/api/contribute/register` | Hub handler-specific | Contribute Proxy | `pkg/hub/server.go:1782` | +| `GET` | `/api/contribute/status` | Hub handler-specific | Contribute Status | `pkg/hub/server.go:1783` | +| `GET` | `/api/contribute/ws` | Hub handler-specific | Contribute WSProxy | `pkg/hub/server.go:1784` | +| `POST` | `/api/github/webhook` | Hub handler-specific | GitHub Webhook | `pkg/hub/server.go:1785` | +| `GET` | `/gh-setup` | Hub handler-specific | GitHub App Setup Router | `pkg/hub/server.go:1786` | +| `GET` | `/learn` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1787` | +| `GET` | `/get-started` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1788` | +| `GET` | `/api/docs` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1791` | +| `GET` | `/api/reading-list` | Hub handler-specific | Reading List | `pkg/hub/server.go:1794` | +| `GET` | `/reading` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1795` | +| `GET` | `/cncf-reference-architecture` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1802` | +| `GET` | `/cncf-reference-architecture/console` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1803` | +| `GET` | `/cncf-reference-architecture/bluefin` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1804` | +| `GET` | `/{$}` | Hub handler-specific | Static HTML page | `pkg/hub/server.go:1821` | +| `GET` | `/og-card.png` | Hub handler-specific | OGCard | `pkg/hub/server.go:1826` | +| `GET` | `/` | Public | Static asset fallback (`http.FileServerFS` over the embedded `static/` tree) for any path no other route claims | `pkg/hub/server.go:1828` | | `GET` | `/activity` | Signed-in user | Fleet activity page ([#16063](https://github.com/hivecommons/hive/issues/16063)): read-only, data-free shell that polls `/api/hub/activity` every 15 s with `If-None-Match`. No session → `307` to `/login`; blocked user → `403`. Own Content-Security-Policy (scripts and styles from `'self'` only, `script-src-attr`/`style-src-attr` `'none'`), `Cache-Control: no-cache` + content `ETag`. See [Hub Fleet activity page](../hub-fleet-activity.md) | `pkg/hub/activity_page.go` | | `GET` | `/activity/public` | Public | Public-safe Fleet activity page ([#15966](https://github.com/hivecommons/hive/issues/15966)): the same data-free shell as `/activity`, served without a session; on this path the page polls `/api/hub/activity/public` with `credentials: 'omit'`. Source of the `twitch.tv/hivecommonsfleet` broadcast. Same CSP and cache headers as `/activity`. See [Hub Fleet activity page](../hub-fleet-activity.md#public-safe-view) | `pkg/hub/activity_public.go` | | `GET` | `/activity/assets/{name}` | Public | The shared tracker assets the Fleet activity page loads, byte-identical: the row asset (`pkg/webassets/trackerrow`, [#16062](https://github.com/hivecommons/hive/issues/16062)) and the `am.*` URL-view and display-playlist helpers (`pkg/webassets/trackerview`, [#16064](https://github.com/hivecommons/hive/issues/16064)); only names in `trackerrow.Files` or `trackerview.Files`, else `404`. `Cache-Control: no-cache` + content `ETag` | `pkg/hub/activity_page.go` | diff --git a/src/docs/api-reference/knowledge.md b/src/docs/api-reference/knowledge.md index b315fc8ed5..391b7c4287 100644 --- a/src/docs/api-reference/knowledge.md +++ b/src/docs/api-reference/knowledge.md @@ -63,3 +63,12 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `GET` | `/api/knowledge/connectors` | Dashboard auth/session | Knowledge connector statuses and publish mirror status (`{"connectors": [...], "publish": {...}\|null}`, see [knowledge-connectors.md](../knowledge-connectors.md)) | `pkg/dashboard/api_knowledge_connectors.go:23` | | `POST` | `/api/knowledge/connectors/{name}/sync` | Owner only | Start one connector sync now (202; 404 unknown, 409 running) | `pkg/dashboard/api_knowledge_connectors.go:24` | | `POST` | `/api/knowledge/publish/sync` | Owner only | Run a publish mirror batch now (202 with publish status; 503 mirror not running, 409 batch in progress) | `pkg/dashboard/api_knowledge_connectors.go:25` | +| `GET` | `/api/knowledge/connectors/{name}/auth` | Dashboard auth/session | Interactive sign-in state for a connector (Plaud OAuth): connected, account, pending, auth_url; `?refresh=1` re-checks upstream (400 for connector types without interactive sign-in) | `pkg/dashboard/api_knowledge_connector_auth.go:25` | +| `POST` | `/api/knowledge/connectors/{name}/auth/connect` | Owner only | Start the connector's sign-in flow (202 with state incl. `auth_url`) | `pkg/dashboard/api_knowledge_connector_auth.go:26` | +| `POST` | `/api/knowledge/connectors/{name}/auth/disconnect` | Owner only | Sign out and revoke the connector's credential upstream | `pkg/dashboard/api_knowledge_connector_auth.go:27` | +| `POST` | `/api/knowledge/connectors/{name}/auth/complete` | Owner only | Finish a sign-in from pasted material on a hosted hive: `{redirect_url}` (the `localhost:8199/auth/callback?…` URL the browser landed on, forwarded in-pod while the login is pending) or `{tokens}` (contents of `~/.plaud/tokens-mcp.json`); see [inbox-voice-sources.md](../inbox-voice-sources.md) | `pkg/dashboard/api_inbox.go:29` | +| `GET` | `/api/inbox` | Dashboard auth/session | Inbox items (voice recordings/dictations keyed by `(source, id)`) with per-destination route state, source connection status, candidate repos and auto-routing settings; see [inbox-voice-sources.md](../inbox-voice-sources.md) | `pkg/dashboard/api_inbox.go:24` | +| `POST` | `/api/inbox/route` | Owner only | Route one item: `{source, item_id, destination: spek_rfc\|dibs\|dismiss, repo?}` (202 queued; 400 unconfigured destination, 404 unknown item, 409 already queued/running) | `pkg/dashboard/api_inbox.go:25` | +| `POST` | `/api/inbox/dismiss` | Owner only | Bulk "Knowledge only": `{items: [{source, item_id}], undo?}` | `pkg/dashboard/api_inbox.go:26` | +| `GET` | `/api/inbox/settings` | Dashboard auth/session | Auto-routing rules, default RFC repo, dibs URL, destination readiness | `pkg/dashboard/api_inbox.go:27` | +| `POST` | `/api/inbox/settings` | Owner only | Update `inbox.rules` / `inbox.spek.repo` / `inbox.dibs.url` in the runtime config (omitted fields unchanged) | `pkg/dashboard/api_inbox.go:28` | diff --git a/src/docs/api-reference/packs-and-acmm.md b/src/docs/api-reference/packs-and-acmm.md index 3fb6bea551..22b5c77ed8 100644 --- a/src/docs/api-reference/packs-and-acmm.md +++ b/src/docs/api-reference/packs-and-acmm.md @@ -25,7 +25,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | `POST` | `/api/repos/{owner}/{repo}/items/{number}/decide-delegated/plan` | Same as `…/decide-delegated` | Preview for *Let hive decide*: runs the same owner-delegate policy with **no side effects** (nothing written, no job) and answers `{would, rule, acting, title, summary, why, consequences[], ladder, mergeable_as_is, alternatives[], writes: false, prior, notes}` — `summary` is one verb-first sentence with no "Hive would:" prefix (*Send #15332 back to its author with one instruction: make CI green, nothing else.*), `why` one short line (*CI is red.*), `consequences` at most two items, `ladder` the rung that decided with its `ruled_out` list, and `mergeable_as_is` whether an as-is merge is honest (the queue-merge and merge-now alternatives are dropped when it is not). The dialog shows **Cancel** / **Let hive do that**, plus **Approve and let hive merge** (or **Merge now as maintainer** when the hive would approve anyway) only when `mergeable_as_is`; only then does the SPA post `…/decide-delegated` with `chosen` naming the button. A transient GitHub failure (5xx, rate limit, timeout) is retried twice with backoff before the plan gives up. A failure answers `{ok: false, error, reason, retryable, attempts, elapsed_ms, hive, hint}` under **409** when the item is no longer open (`reason: not-open`) or **424** when a read it depends on failed (`github`), is not configured (`config`) or the plan itself failed (`plan`) — never 502/503, which the hosted Ingress intercepts and replaces with its generic *hive unavailable* page (hivecommons/hive#16776). See [dashboard-sections.md](../dashboard-sections.md#let-hive-decide). | `pkg/dashboard/api.go:378`, `pkg/dashboard/api_repo_item_decide_delegated_plan.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/recheck` | Same as `…/hold` (owner or repo write), signed-in GitHub user | *Re-check permissions* on an issue parked on a capability blocker (#15099): probes the GitHub App installation's permissions; when the missing permission is granted, clears every ledger entry blocked on it and un-parks the item (`{result: {outcome: "released", …}}`), otherwise `{result: {outcome: "no-effect", reason: "blocker", options: [...]}}`; 404 when the item has no blocker, 409 when the blocker is not a capability gap | `pkg/dashboard/api.go:379` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/blocker/close-partial` | Same as `…/hold` (owner or repo write), signed-in GitHub user | Option (c) on a capability blocker (#15099): labels the issue `hive/partially-addressed`, posts the operator's `{"comment"}` naming what remains and closes it; the ledger entry is cleared | `pkg/dashboard/api.go:381` | -| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:592`, `pkg/dashboard/api_repo_item_accept_triage.go` | +| `POST` | `/api/repos/{owner}/{repo}/items/{number}/accept-triage` | Owner or repo write/maintain/admin | Accept triage on an issue from the activity monitor's row menu — what a maintainer accepting triage on GitHub does by hand: adds the reporter-trust triage label (`triage/accepted`, or the first of `issue_filter.reporter_trust.untrusted_require_labels`; the label is created in the repo if missing) and removes the awaiting label (`needs-triage` / `reporter_trust.awaiting_label`) and a Hive-written `stage/015-triage-needed`. Empty body. Returns `{ok, repo, number, acceptLabel, added, removed, alreadyAccepted, minStatusSeq}`; an issue already carrying the accept label with nothing left to remove is a no-op (`alreadyAccepted: true`, no write, no audit). 404 for a pull request or an unknown item. The status snapshot's labels are patched so the next reconcile reads 010 Actionable before the engine's own 015 → 010 derivation. Every write is audited as `repo_item_accept_triage` under the dashboard user. | `pkg/dashboard/api.go:594`, `pkg/dashboard/api_repo_item_accept_triage.go` | | `POST` | `/api/repos/{owner}/{repo}/items/{number}/sentinel-confirm` | Owner or repo write/maintain/admin | **Confirm safe** on the activity monitor's human-gate pill ([#17470](https://github.com/hivecommons/hive/issues/17470)): a person confirms a sentinel finding is intended. Body `{comment?, type?}` (`type` is `pr` or `issue`, comment ≤ 4000 characters). Posts the confirmation as a comment first — ``, `@user confirmed the sentinel finding safe from the dashboard.`, then the comment text — and removes every label matching the configured sentinel label (`sentinel.label`, default `sentinel-alert`, case-insensitive). Returns `{ok, repo, number, label, removed, alreadyClear, result}`; with no sentinel label on the item the comment is still posted and `alreadyClear` is true. 403 for a repository the hive does not watch. Audited as `repo_item_sentinel_confirm` under the dashboard user; the operator-action ledger records it as `confirm-sentinel`. | `pkg/dashboard/api.go:369`, `pkg/dashboard/api_repo_item_sentinel_confirm.go` | | `GET` | `/api/acmm/evaluation` | Dashboard auth/session | ACMMEvaluation — combined codebase + operational result, cached server-side for 1 hour (`acmmEvalTTL`). `?refresh=1` (the dashboard's "🔄 Re-evaluate" button, #5877) bypasses the hourly TTL but is debounced server-side: requests within 1 minute of the last evaluation (`acmmRefreshDebounce`) still serve the cache, since a full refresh costs up to ~29 GitHub GetContents calls per repo. The response's `last_evaluated_at` timestamp reports when the cached evaluation was computed | `pkg/dashboard/api.go:383`, `pkg/dashboard/api_acmm_eval.go` | | `POST` | `/api/acmm/issue` | Owner only | ACMMCreate Issue — files on GitHub or, with `governor.acmm.issue_tracker: work_source` / body `tracker: "work_source"` on a Linear-sourced hive, on Linear; response `tracker` says which. See [ACMM policy matrix](../acmm-policy-matrix.md#where-acmm-gap-issues-are-filed) | `pkg/dashboard/api.go:384` | diff --git a/src/docs/api-reference/snapshots-and-style.md b/src/docs/api-reference/snapshots-and-style.md index 023b887f8f..5bddc23ff2 100644 --- a/src/docs/api-reference/snapshots-and-style.md +++ b/src/docs/api-reference/snapshots-and-style.md @@ -6,7 +6,7 @@ Part of the [Dashboard REST API reference](../api-reference.md); see that page f | Method | Path | Auth | Purpose | Source | |---|---|---|---|---| | `GET` | `/api/style` | Public | Sanitized custom dashboard CSS | `pkg/dashboard/api.go:48` | -| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1933` | +| `GET` | `/branding/custom.css` | Dashboard auth/session | Operator branding stylesheet override, read per request (see [branding](../branding.md)) | `pkg/dashboard/server.go:1944` | | `GET` | `/api/snapshot/frame-ancestors` | Public | Snapshot framing allowlist | `pkg/dashboard/api.go:91` | | `GET` | `/api/snapshot` | Public | Snapshot data | `pkg/dashboard/api.go:92` | | `GET` | `/snapshot` | Public | Public read-only snapshot page | `pkg/dashboard/api.go:93` | diff --git a/src/docs/benchmarks.md b/src/docs/benchmarks.md index be9cf68661..ef2ef3c072 100644 --- a/src/docs/benchmarks.md +++ b/src/docs/benchmarks.md @@ -28,11 +28,28 @@ One record per PR per role, conforming to | `task.acmm_level` | `policy.acmm_level` at review time. | | `outcome.*` | `pr_opened`, `pr_merged`, `merged_at`, `time_to_merge_s`, `ci_status_at_merge`, `ci_green_first_try`, `review_verdict`, `reverted`, `closed_unmerged`, `human_interventions` (human approvals / label changes). | | `quality.*` | Swarm verdict counts and confidence. | +| `cost.*` | Estimated spend — see [Cost](#cost-estimated-never-billed). | **What never leaves the spoke:** the hive id, repository and organization names (unless the repository is public — see below), PR numbers, commit SHAs, -agent prompts, diff content, reviewer comments, human logins, and cost/token -figures (not present in bundles today). +agent prompts, diff content, reviewer comments, human logins, per-session +token usage and any billed figure. Only the per-PR *estimated* USD and its +token share leave, rounded by the price table. + +## Cost: estimated, never billed + +Each record carries `cost.basis` and, when attributable, `cost.usd_est`: + +| `cost.basis` | Meaning | +|---|---| +| `tokens_x_pricing` | The spoke's launch-cost collector (`/api/launch-cost`, `/data/launch-cost.json`) saw a kick that named this PR — the swarm's `owner/repo#N` PR ref for reviewer records, or the governor's `repo#N` issue ref of an issue the PR closed (`issue_closures`) for author records. The kick's token usage is priced with `pkg/tokens/pricing.go` (public list prices, `price_table_date`) and split evenly across the refs the kick named — the same `share_usd` the Cost panel shows. `launches` counts the kicks folded in; `*_tokens` are the in-window token share, informational. | +| `unknown` | No kick with usage named the PR: kicked without a ref, older than the launch-cost store, or usage landed outside the 6 h ownership horizon. | + +The spoke never emits `fleet_model_median`; the benchmarks site imputes that +at index-build time from attributed records of the same model and role and +labels it as such. `reported` (a gateway-billed figure) is reserved. List +prices are not what subscription plans or self-hosted inference cost, so the +site shows every dollar as an estimate with its basis mix. ## Privacy model: default on, anonymized @@ -97,5 +114,6 @@ pods and a nightly workflow rebuilds the index. - `pkg/evidence/benchmarks` — record types, `HiveHash`, `Convert` (bundles → records), schema golden test. - `pkg/dashboard/api_benchmarks.go` — export endpoint, status block, hourly pusher, Settings panel data. +- `pkg/dashboard/benchmarks_cost.go` — `CostLookup` over the launch-cost collector (daily rollup USD + snapshot tokens). - `pkg/hub/benchmarks.go` — ingest, month-file store, hourly GitHub publisher, status endpoint. - `pkg/evidence` `Author.runtime` / `Author.task_kind` — stamped at bundle write via `github.EvidenceAgentContext`. diff --git a/src/docs/dashboard-sections.md b/src/docs/dashboard-sections.md index 2b4ea104c1..ce81c8151c 100644 --- a/src/docs/dashboard-sections.md +++ b/src/docs/dashboard-sections.md @@ -189,6 +189,11 @@ Full entry: [Knowledge](dashboard-sections/workflow-and-agents.md#knowledge). Find past pull requests, issues, review records and findings across your projects by keyword, with a link to each one. Full entry: [Search](dashboard-sections/workflow-and-agents.md#search). +## Inbox + +Voice recordings and dictations waiting for you to file them as an RFC, send to dibs, or keep as knowledge. + +Full entry: [Inbox](dashboard-sections/workflow-and-agents.md#inbox). ## Contributors diff --git a/src/docs/dashboard-sections/workflow-and-agents.md b/src/docs/dashboard-sections/workflow-and-agents.md index 14a92427ec..008f5fd31b 100644 --- a/src/docs/dashboard-sections/workflow-and-agents.md +++ b/src/docs/dashboard-sections/workflow-and-agents.md @@ -205,6 +205,21 @@ The facts Hive has learned about your projects, where they came from, and how mu **What it is good for.** Check it when a worker keeps making the same mistake. Correct the wrong fact here. +## Inbox + +Voice recordings and dictations waiting for you to file them as an RFC, send to dibs, or keep as knowledge. + +**What it tells you.** Every recording a voice source (Plaud, Wispr Flow) syncs is already a Knowledge fact. The Inbox lists the ones you have not triaged yet, with the source, date, length and a short excerpt. + +**How the numbers are worked out.** + +- The sidebar count and the summary count items with no destination yet and not marked "Knowledge only". +- Each item shows one pill per destination it was sent to: queued, running, filed (with a link) or failed. +- Auto-routing rules run once per new item after a sync; the first matching rule wins. +- The default RFC repository, dibs URL and rules can also be edited under Settings → Inbox. + +**What it is good for.** Record an idea on your recorder, then turn it into an RFC with child issues (→ RFC), a dibs idea (→ dibs), or just keep it searchable (Knowledge only) without leaving the dashboard. See [Inbox and voice sources](../inbox-voice-sources.md). + **Example.** A fact says tests run with `make test`, but your project uses `go test`. You edit the fact. **When it appears.** Always. diff --git a/src/docs/documentation-map.md b/src/docs/documentation-map.md index 3693b2908a..a7270562b1 100644 --- a/src/docs/documentation-map.md +++ b/src/docs/documentation-map.md @@ -174,6 +174,7 @@ Documentation for the current Hive line (branch `v5`; the code and docs live und - [Portable AgentDefinition format](../AGENT-DEFINITION.md) — standalone YAML schema for importing/exporting agent definitions. - [How Hive injects knowledge and memory into agents](agent-knowledge-and-memory.md) — the three context layers (Hive knowledge primer, CLI conversation, CLI persistent memory), the kick-primer path and its scoping/caps, why repo `AGENTS.md` is injected rather than read natively, which per-agent home directories are fleet-shared, and what survives a kick, `/clear`, pod restart or a new agent. - [Knowledge connectors](knowledge-connectors.md) — **Owner configuration:** connect, validate, and sync external document/wiki sources; includes scope and API reference. +- [Inbox and voice sources](inbox-voice-sources.md) — triage Plaud/Wispr Flow recordings into spektacular RFCs, dibs ideas, or knowledge only; hosted-hive Plaud sign-in. - [Knowledge curator](knowledge-curator.md) — configure fact extraction, sources, vaults, document imports, code maps, and retention. - [Knowledge confidence scoring](knowledge-confidence.md) — interpret confidence badges and score explanations. - [Knowledge lifecycle states](knowledge-lifecycle.md) — approve, deprecate, or supersede facts and keep stale knowledge out of prompts. diff --git a/src/docs/fleet-report.md b/src/docs/fleet-report.md index e00f395026..beecf18ad7 100644 --- a/src/docs/fleet-report.md +++ b/src/docs/fleet-report.md @@ -142,7 +142,7 @@ title is `[selfreport] : `. | `agent_crash_loop` | the watchdog latches an agent as crash-looping and pauses it (acting mode only — observe mode would re-count every sweep); the healthy-window reset clears it. Diagnostics carry the classifier detail, failure count/cap, backoff ladder and healthy-reset window (#13470, was "agent crash loop") | high | | `ci_poll_nudge` | the harness sent a non-monitoring agent the stop-polling-CI nudge (at most one per kick); reviewer, CI-maintainer and quality lanes are exempt because reading CI status across PRs and repositories is their assigned work. Only a command line run more than once since the kick counts as polling (#14016): a sweep that reads each of several PRs' checks once does not cross the count threshold. A self-report fires only after an eligible agent ran ≥2 such repeated polls since the kick — a single nudged poll or a sweep does not trigger a report. One per 24h window; message carries polls since kick / time since kick / nudges since boot, diagnostics the scrubbed pane tail plus the repeated CI-status command lines with their counts (#14289; #13470, was "CI-poll nudge") | low | | `issue_request_stuck` | an issue/comment/label request file in the relay queue (`issue-requests/`) has failed ≥3 attempts **or** has been retrying for ≥30 min. Key is the request file name; message carries kind, attempts and age, diagnostics the last error (scrubbed). Clears when the request completes or is quarantined (`.failed`/`.bad`) | medium | -| `effect_ledger_op_stuck` | the mutation boundary refused a retry because the journaled external effect (issue create, PR open, …) has been `Unknown`/`Planned` for ≥30 min — the caller keeps replaying the same logical op and it never resolves. Key is the op's logical ID; message carries kind, status and age, diagnostics the attempt count, repo and claim target. Clears when the op is applied, replayed as already-applied, or reconciled | medium | +| `effect_ledger_op_stuck` | the mutation boundary refused a retry because the journaled external effect (issue create, PR open, …) has been `Unknown`/`Planned` for ≥30 min — the caller keeps replaying the same logical op and it never resolves. Key is the op's logical ID; message carries kind, status and age, diagnostics the attempt count, repo and claim target. Clears when the op is applied, replayed as already-applied, reconciled, or — for `issue_create` — expired to NotApplied once a refused retry finds it unresolved past 1h (the caller's next attempt re-runs its duplicate search, then creates; #17693) | medium | | `agent_dir_setup_failed` | the harness failed to set up the per-agent sandbox directory structure — create XDG directories, restore or tighten directory permissions on interactive/inference home directories. Log-fed: keywords include `failed to create per-agent xdg dir`, `failed to restore per-agent dir mode`, `failed to restore interactive home mode`, `failed to tighten per-agent dir mode`, `failed to tighten interactive home mode`, `failed to restore inference home mode`, `failed to tighten inference home mode` (`crux/agent/manager.go`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `agent_pane_timeout` | the agent-readiness wait timed out and the harness dumped the tmux pane (`prompt timeout — dumping pane` / `pane content at timeout`, `crux/agent/manager.go`), and the startup kick dropped because the CLI never became ready or reached its input prompt (`startup kick dropped: …`, `crux/agent/manager_kick.go`, the same event as the pane dump). Log-fed: same ≥5-in-10-min repeat threshold and 20-min idle clear as the catch-all below; pane excerpts are scrubbed and bounded before storage | low | | `agent_login_required` | the login scanner saw the agent CLI asking for `/login` (`login required detected`, `pkg/loginscan`) — the agent's auth lapsed and it is paused until an operator re-authenticates. The record carries `backend` (which provider to re-authenticate), `login_hint` (the CLI's login command) and `credential_valid`; re-login in the agent's tmux pane, then Resume. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | @@ -150,13 +150,18 @@ title is `[selfreport] : `. | `bead_fact_ingest_failed` | the knowledge system failed to ingest a bead fact from an agent (`failed to ingest bead fact`, `pkg/knowledge`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `feedback_issue_refresh_failed` | the dashboard's feedback-issue watcher failed to refresh or update a feedback issue (`feedback: issue refresh failed`, `pkg/dashboard`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `github_app_repo_not_covered` | the GitHub App recheck (`state=repo-not-covered`) found the installation does not include every configured repo — an operator configuration condition (tick the repos in the App's repository access), not a hive defect. Log-fed: same ≥5-in-10-min repeat gate and 20-minute idle clear as `agent_pane_timeout` | low | +| `github_app_alerts_permission_missing` | the sec-check scoped token mint was refused with Dependabot alerts read requested and retried without it. The App owner must grant Dependabot alerts read-only and re-accept it on the installation; the fallback token cannot read alerts. Log-fed: ≥5 warnings in 10 minutes, clears after 20 quiet minutes. See [GitHub App setup](github-app-setup.md#adding-a-permission-to-an-existing-app) | medium | +| `github_app_statuses_permission_missing` | GitHub answered the queue action's `prow/lgtm` commit-status write with 403 (`could not set the prow/lgtm commit status: the GitHub App lacks the statuses: write permission`, `pkg/github/client.go`): the App installation lacks Commit statuses read & write, so Prow may strip the `lgtm` label and the auto-merge queue may not hold. The App owner must grant Commit statuses read & write and re-accept it on the installation. Only the 403 carries this wording — any other status failure (5xx, 422, transport) logs the neutral `could not set the prow/lgtm commit status` and is not this class. Log-fed: ≥5 warnings in 10 minutes, clears after 20 quiet minutes. See [GitHub App setup](github-app-setup.md#adding-a-permission-to-an-existing-app) | medium | +| `hub_heartbeat_rejected_unprovisioned` | the hub answered the heartbeat 403 `hosted/saas hive IDs can only be created via provisioning` — it does not know this hive ID (deprovisioned, renamed or registry drift), so retrying cannot succeed. The hive backs off 10 minutes between probes; fix by provisioning or re-IDing the hive. Log-fed, fires on first sighting, 3-hour idle clear | medium | | `github_app_not_installed` | a per-agent scoped-token mint was attempted while the GitHub App has no installation ID (`installation_id` 0): the App is not installed on the hive's org, or discovery never found it. An operator configuration condition — install the App or set `github.installation_id` — not a hive defect. Key is the agent name; reported on the first observation and cleared by the next successful mint. The runtime logs the matching WARN once per agent instead of every supervisor tick (#14186) | medium | +| `forge_unreachable` | a per-agent scoped-token mint got no answer from the GitHub host (dial timeout, connection refused, context deadline — `isForgeTransportErr` in `crux/agent/manager_tokens.go`): egress from the pod to the forge is broken. Nothing about the App key or installation follows from it, so it is reported apart from `github_app_token_mint_failed` and clears that class if it was open. Key is the agent name; cleared by the next successful mint. Fix cluster egress, DNS or the proxy/`hostAliases` in front of the forge, not the App | medium | | `ioscan_classifier_unavailable` | the ioscan model classifier failed closed (`ioscan classifier failed closed; text withheld`, `pkg/scheduler/ioscan_enforce.go`), withholding untrusted text, or a genuine backend error under `ioscan.classifier.fail_mode: open` passed text on unclassified (`ioscan classifier failed open; text passes on unclassified`). Under fail-open, a missing model endpoint instead logs `ioscan classifier not configured; semantic scan skipped (fail-open)`; semantic scanning is skipped and this operator configuration condition is not fleet-reported. A missing endpoint under fail-closed still withholds text and is reportable. The scheduler logs `ioscan fail-closed blocked kick` (`pkg/scheduler/templates.go`) or `ioscan fail-closed blocked held-PR coordination kick` (`pkg/scheduler/policy_overlays.go`) when it withholds a kick for the same reason, which maps to this class too. The WARN is already deduped per failure by `warnOnce` (once per process), so it is reported on the first observation (`MinCount` 1) with the normal idle clear; this is a real degradation, not noise. Log-fed | medium | | `github_fetch_transient` | a non-critical GitHub fetch failed and the feature degrades gracefully (`channel commit date: fetch failed — the row will render without a timestamp`, `pkg/hub/channel_commit_date.go`) — typically a rate limit or transient API error, not a hive defect. Reported only when persistent: log-fed, same ≥5-in-10-min repeat threshold and idle clear as `agent_pane_timeout`. The sibling `SHA poll: commit message fetch non-200` has its own class, `sha_poll_commit_message_unavailable` | low | | `label_provisioning_failed` | provisioning the issue stage labels on a repo failed (`issue stage labels: provisioning failed`, `pkg/github/issue_stage_labels.go`) — typically the App lacking label permission on that repo or a transient API error; diagnostics carry the repo and scrubbed error. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `agent_cli_restart_loop` | the agent CLI crashed (bare shell) or sat on a consent screen and the harness restarted it before the kick (`agent CLI crashed or stuck on consent screen, restarting before kick`, `crux/agent/manager_kick.go`, `kick_async.go`), repeated ≥5 times in 10 min — a restart loop, not transient noise. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `copilot_cli_hung` | a Copilot agent showed none of the CLI's startup chrome (input prompt, help hint, login request) 180 s after launch, outside kick delivery, so the hang detector killed its session, ran bare `copilot` to capture the error and relaunched the agent (`copilot hung with no CLI prompt, running diagnostic`, `crux/agent/manager_poll.go`), or that diagnostic saw no output before its own deadline and relaunched anyway (`diagnostic: timed out waiting for copilot error output`, `crux/agent/manager_credentials.go`; the line carries the pane's `output_tail`). The key is the message, so ≥5 in 10 min counts every Copilot agent on the hive together: its Copilot agents keep failing to reach their prompt. Typical causes are a stale Copilot token the CLI hangs on (the diagnostic restores or clears it when it sees an auth error), a blocked network path to the Copilot API, or a CLI build whose startup screen the detector does not recognise (`cliReadyIndicators`, `crux/agent/pane_classify.go`). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `proxy_connect_dial_failed` | the hive proxy could not dial the upstream host of a CONNECT request (`proxy: CONNECT dial failed`, `crux/proxy/github_proxy_connect.go`) or of a Copilot usage-sniffed connection (`copilot sniff: upstream dial failed`, `crux/proxy/github_proxy_copilot.go`) — typically DNS or network unreachability of an agent-requested host (for example a dial timeout to `telemetry.enterprise.githubcopilot.com`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | +| `proxy_plain_forward_failed` | the proxy could not complete an agent's plain-HTTP upstream request (`proxy: plain HTTP forward failed`, `crux/proxy/github_proxy_connect.go`), returning HTTP 502. Check the requested host and logged error, including DNS and outbound connectivity; this differs from a destination refused by the proxy policy. Log-fed: ≥5 occurrences in 10 min, clears after 20 quiet minutes | medium | | `proxy_tls_handshake_failed` | the hive proxy's TLS handshake with an upstream host failed (`proxy client tls handshake failed`, `transparent proxy tls handshake failed`, `copilot sniff: client tls handshake failed`, `crux/proxy/github_proxy_connect.go`, `crux/proxy/github_proxy_copilot.go`) — typically a certificate validation failure or encryption mismatch on an agent-requested host, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `review_evidence_write_failed` | the github-client failed to write a review evidence bundle to persistent storage (`review evidence: could not write bundle`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `selfreport_cleared_comment_failed` | the fleet report publisher failed to post the "condition cleared" comment on a self-report issue (`fleet report: cleared comment failed`, `pkg/github`), repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | @@ -165,6 +170,7 @@ title is `[selfreport] : `. | `inference_upstream_error` | a failed or non-2xx response from the configured inference backend (LiteLLM, vLLM/llm-d) on the Anthropic-to-OpenAI translator or MITM reroute path (`inference upstream error` / `inference upstream failed`), or a request the translator could not translate (`inference translate request failed`, logged at ERROR by `logTranslateFailure`), all in `crux/proxy/github_proxy_inference.go`, repeated ≥5 times in 10 min — covers transport failures and statuses other than the dedicated 401/403 auth signal. A request refused because the agent has neither an agent model nor a gateway `default_model` logs `inference model not configured; request refused` at WARN instead and is not counted here (a benign configuration condition; see below). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `proxy_plain_forward_refused` | the hive proxy refused a plain-HTTP forward to a non-globally-routable destination (`proxy: plain HTTP forward refused`, `crux/proxy/github_proxy_connect.go`) — a client ignored its proxy bypass for a private address (e.g. the Docker bridge gateway). Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: fix the client's `NO_PROXY` or target address; the refusal is counted as a proxy violation | low | | `sha_poll_commit_message_unavailable` | the SHA poller's commit-message fetch got a non-200 from GitHub (`SHA poll: commit message fetch non-200`, `pkg/hub/saas_sha_poller.go`). The poller keeps serving the SHA without a message. A throttled 403/429 is treated as client-wide: every SHA is parked until `Retry-After` / `X-RateLimit-Reset` elapses and a single WARN is logged per hour per status (`warnGitHubFetchOnce`, key `sha-msg:throttled:`), so a rate limit or revoked token cannot trip this class. Other statuses (e.g. 404 for a force-pushed-away commit) back off per SHA and are deduped to once per hour per (SHA, status), so one stuck commit logs one line; the class is reported only when ≥5 distinct SHA/status pairs fail within 10 min (many branches whose commits GitHub no longer serves). Its main job is keeping the residual WARN out of `unclassified_error_repeated`. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | +| `agent_lifecycle_repeated` | known agent-lifecycle log lines (`failed to send kick`, `agent CLI crashed (bare shell detected)`, `tmux kill-session before restart failed`, `agent tmux session missing`, watchdog `would restart dead agent` in observe mode) repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout`; replaces the catch-all `unclassified_error_repeated` for these lines | low | | `tmux_session_check_failed` | the agent's tmux session `has-session` check failed (`tmux has-session failed`, `crux/agent/manager.go`) — typically the session was lost or the tmux server is unhealthy, repeated ≥5 times in 10 min. Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | medium | | `agent_github_api_cap_reached` | an agent exhausted its per-agent GitHub API hourly budget (`agent GitHub API hourly cap reached`, `crux/proxy/agent_github_budget.go`; fields `agent`, `count`, `cap`, `retry_after`). The proxy throttles further calls until the window resets, so this reports sustained polling by one agent rather than an unknown error. Log-fed | medium | | `transcript_scan_slow` | a Copilot transcript activity scan took longer than its interval (`transcript activity scan outlived its interval`, `crux/agent/busy_visibility.go`) — agent liveness reads may lag. The WARN itself is debounced at the source (`transcriptScanPace`, `crux/agent/transcript_scan_pace.go`): it fires once per slow episode, only after 3 consecutive passes outlive the interval, and re-arms only after a pass finishes under 80% of the interval, so one sustained overrun logs one line and never trips this class; it is reported only when ≥5 distinct slow→fast→slow episodes occur within 10 min. Log-fed: same ≥5-in-10-min repeat threshold and 20-minute idle clear as `agent_pane_timeout`. Operator diagnostics: check the agent's transcript/session volume; it is bounded by concurrency, so repeated episodes indicate slow disk or very large session directories | low | @@ -172,11 +178,21 @@ title is `[selfreport] : `. | `models_endpoint_empty` | the dashboard's inference model discovery (`/api/inference/{backend}/models`) found no models from any configured endpoint (`no models found from any endpoint`, `pkg/dashboard/api_models.go`) and fell back to the static aliases, flagged unverified in the UI. The source logs it at WARN once per backend per hour (DEBUG on repeats, re-armed when discovery succeeds), so this fires only when the condition flaps ≥5 times in 10 min (gateway key rejected, endpoint down). Log-fed: same repeat threshold and idle clear as `agent_pane_timeout` | low | | `unclassified_error_repeated` | **generic catch-all**: the same WARN/ERROR log line (numbers, hashes, paths and quoted strings collapsed) repeated ≥5 times in 10 min. Fed by a `slog.Handler` tee (`pkg/fleetreport/slogfeed`) on the hive's root logger, so a repeating defect is noticed even with no hook site. Skips benign lines (read/keepalive i/o timeouts, WARNs whose `error` is a GitHub rate-limit refusal or a 502/503/504 gateway failure — the caller retries next tick, context cancellation, the permissions watcher's `fixed …` self-healing lines) and any message that already matches another class's keywords. Clears automatically after 20 quiet minutes (2× the window), so a "cleared" comment follows once the symptom stops | low | +The cancellation exclusion includes structured `error` attributes, including +logger-bound attributes: typed errors wrapping `context.Canceled`, exact +`context canceled` strings, and strings ending in `: context canceled`. +Warnings remain in local logs; deadlines and other errors remain reportable, +including when another `error` attribute accompanies a cancellation. + The GitHub rate-limit exclusion also covers the local REST reserve guard: when optional issue activity or CI trail fetches are shed below the reserve, their warnings remain in local logs but do not create repeated-error reports. Issue activity keeps its last rollup and is retried by the collector. +Canceled maintainer-edit access reads during an in-place adoption sweep stay in +local logs without triggering the catch-all; adoption is preserved and the +read is retried on the next sweep. Other access-read errors remain reportable. + Callers that fail closed on a GitHub HTTP 500 from the combined-status endpoint — the merge-eligible lane gate, the trusted-author and contributor-lane automerge sweeps, and the stage-label check digest — also stay in the logs @@ -236,6 +252,16 @@ nudge-delivery failures remain eligible for reporting when repeated. It also skips `stage transition suppressed by dampener` (`stageDampenerSuppression`, `pkg/fleetreport/slogfeed/slogfeed.go`): the stage-label store suppresses flapping transitions during its dampener hold window (hivecommons/hive#17103). That is the dampener working as designed, not a runtime error. The same function skips `stage flap suppressed` (`suppressStageFlap`, `pkg/stageengine/writer.go`): the stage engine refuses a same-head, same-evidence reversal of its last label write inside its flap window (hivecommons/hive#17142). +It also skips the exact WARN `signed-commit reconciler: skipped; the PR keeps an +unsigned commit` (`signedCommitOwnershipSkip`, `pkg/fleetreport/slogfeed/slogfeed.go`; +emitted by `pkg/github/pr_signed_reconcile.go`): the signed-commit reconciler +deliberately refuses to rewrite a merge commit or a commit authored or committed +by anyone other than a hive agent, and records the remedy on the PR +(`noteSignedSkip`). Repeated refusals are the ownership guard working as +designed, not a repeated runtime defect (hivecommons/hive#17924). The WARN and +its `reason` stay in local logs; other reconciler failures and the same message +at ERROR remain reportable. + The re-engagement sweep logs `re-engagement kick undeliverable; budget not charged` at INFO rather than WARN when the cause is `shared session pool is busy` (`deliverReEngagement`, `src/cmd/hive/main_sweeps.go`): a saturated shared CLI diff --git a/src/docs/github-app-setup.md b/src/docs/github-app-setup.md index 4e713f4365..2caef7ed63 100644 --- a/src/docs/github-app-setup.md +++ b/src/docs/github-app-setup.md @@ -217,11 +217,12 @@ that has not approved a permission update is the one that still looks fine, and GitHub keeps an App on its old permissions until an org owner accepts, so a partly approved fleet is otherwise invisible. -Verdicts are computed where a GitHub call has failed and on the dashboard's -**Re-check** button, not on every eval cycle, so this adds no API calls to a -healthy hive. To read a specific installation's grants on demand — for example -to confirm the demo/canary install — press Re-check on that hive and read the -line above from its logs. +Permission verdicts are computed where a GitHub call has failed and on the +dashboard's **Re-check** button, not on every eval cycle. Repository coverage +is checked separately on the self-heal loop, including on a healthy hive, as +described below. To read a specific installation's grants on demand — for +example to confirm the demo/canary install — press Re-check on that hive and +read the line above from its logs. ## Install and configure Hive @@ -265,7 +266,9 @@ What it does not do, and falls back on (the PR still opens on the agent's own co **Follow-up commits.** An agent's later pushes to its open PR (a CI fix, review follow-ups, a rebase) are plain git and unsigned, and so is a PR that opened on a `signed_skipped` fallback. With `app_signed_commits` on, the PR-request watcher also runs a reconcile pass, at most once a minute (#9364). A PR needs signing when *any* of its commits is not **Verified**, not only its head: a person can sign their own commit on top of an agent's unsigned one, which leaves the head Verified while the PR is still blocked (#9531). For every open PR authored by the App bot, the pass finds the first unverified commit (oldest to newest) and re-authors everything from there through the head as one GitHub-signed commit on top of the commit right before it. The tree is the same, and the message carries the range's messages and DCO trailers, re-addressed as above. When the PR's first commit is the unverified one, it signs against the PR's base branch, as at open time. A fully Verified PR is left alone. The open path's guards all still apply. A head that moves mid-rewrite is retried on the next pass, and so is a head GitHub could not be reached for (a 5xx or network error). A head GitHub *refuses* to sign (a 4xx answer, such as the 422 on the scratch ref in #16193) is retried on three consecutive passes and then settled, with the `hive:signed-commits-blocked` comment quoting the error; there is no person's commit to squash in that case, so the remedy is to push a new head (which restarts signing) or fix the repository-side cause the error names. A PR whose head hasn't changed since the last pass costs no calls beyond the open-PR list. -The pass re-signs only the hive's own work. Every commit in that range, Verified or not, must have an author and committer that are the App bot or an address in the pane identity domain (`HIVE_GIT_BOT_EMAIL_DOMAIN`, default `hive.kubestellar.io`). Verified App-bot commits in the range are folded in. A person's commit (even a Verified one on top), or a merge commit, leaves the branch as pushed. So does any change the mutation can't express. The PR then gets one comment (marked ``, posted once per PR) naming the unsigned commit and the commit that blocks the rewrite, so it can't merge under `required_signatures` until someone acts. The manual fix is to squash the unsigned commit into a commit the person signs, or to drop the person's commit so the hive can re-sign the range and then push it again, signed, on top. +The pass re-signs only the hive's own work. Every commit in that range, Verified or not, must have an author and committer that are the App bot or an address in the pane identity domain (`HIVE_GIT_BOT_EMAIL_DOMAIN`, default `hive.kubestellar.io`). Verified App-bot commits in the range are folded in. A person's commit (even a Verified one on top), or a merge commit, leaves the branch as pushed. The PR gets one comment (marked ``, posted once per PR) naming the blocker. To unblock it, squash the unsigned commit into a commit the person signs, or drop the person's commit so the hive can re-sign the range and then push it again, signed, on top. + +Changes the signing mutation cannot safely express also leave the branch untouched and get a single comment with the reason. For example, it cannot preserve executable files, symlinks or submodule pointers. A contributor can re-author the unsigned commits with their own verified signature, preserving their contents and modes, and push the signed branch. These deliberate skips are logged at INFO rather than as API failures; a new head is examined again. **Agents after a push.** The rewrite replaces the agent's commits on the remote, so its local branch no longer matches, even though the tree is identical. Before committing again on a branch with an open PR, run `git fetch && git rebase origin/`. The rebase drops the now-empty local commits and keeps any new work. Never force-push over the signed commit: that puts the unsigned commits back, and the next pass has to sign them again. @@ -321,6 +324,22 @@ installation rather than creating a new one, so the `installation_id` in that redirect is one the hive already holds — the failed page does not mean App authentication is broken. +An installation can authenticate successfully while excluding a private +repository under **Only select repositories**. When adding a repo in the +dashboard, use the reported installation settings link to add it to the +selection, then re-check access. For repos already configured, the Forge App +banner identifies missing repository coverage; update the installation and +re-check it. A rejected advisory issue creation can then be retried without +waiting for the next UTC day. + +Boot coverage checks have a time limit; a timeout is inconclusive. The +background self-heal loop rechecks installation coverage every two minutes, +including on a healthy hive, using the cached installation token. Each check +walks up to 10 pages of `GET /installation/repositories`. If that bounded +listing is incomplete, the dashboard checks the repository's App installation +directly. Temporary GitHub errors still block adding the repository until a +check succeeds. + ## `/gh-setup` flow When the app's Setup URL points at `https:///gh-setup`, GitHub redirects back with `setup_action` and, for install/update, `installation_id`. diff --git a/src/docs/good-first-issue.md b/src/docs/good-first-issue.md index f823851f22..2beb94d7e8 100644 --- a/src/docs/good-first-issue.md +++ b/src/docs/good-first-issue.md @@ -121,6 +121,11 @@ wins, oldest first on ties. Issues with the `good first issue` label the hive did not apply are still given the welcome comment (once) so that their visitors get the relay pitch too. +If GitHub refuses a curation write because a repository is archived and +read-only, the sweep skips that repository until hive restarts. Remove archived +repositories from the configured repo list, or restart hive after unarchiving. +Other write failures are still reported and retried on later sweeps. + ## The welcome comment One comment per issue, found by its first-line marker ``. A diff --git a/src/docs/hub-deployment.md b/src/docs/hub-deployment.md index 6fe300c9ff..5090ba7a1a 100644 --- a/src/docs/hub-deployment.md +++ b/src/docs/hub-deployment.md @@ -103,6 +103,8 @@ For push-reported clusters where the hub cannot run `kubectl`, follow the manife ### Push-reported cluster entries +`/data/saas/clusters.json` also accepts per-cluster network overrides for hives it provisions: `upstream_proxy` (`{url, hosts[], no_proxy[]}`, a CONNECT forward proxy, injected as `HIVE_UPSTREAM_PROXY_*` env) and `host_aliases` (`[{ip, hostnames[]}]`, rendered as Pod `hostAliases` for a GHE reachable only via a TLS reverse proxy on a fixed IP; existing hives are converged by a rate-limited reconcile sweep). See [network-requirements.md](network-requirements.md). + `/data/saas/clusters.json` keeps the historical compatibility key `pull_only`. Set `pull_only: true` when the hub cannot reach that cluster's Kubernetes API and the spokes must dial out to the hub instead. Operator-facing surfaces call these clusters **push-reported** because their node stats, dashboard token proofs, and liveness arrive in spoke heartbeats pushed from the cluster. A push-reported entry still needs its stable identity (for example `id`, `name`, `domain`, app/forge fields), but it does not need a usable `kubeconfig_path` for hub-side reads. If node stats are absent, upgrade the spokes to a build that sends heartbeat cluster health and then check the error reason shown in the Clusters panel. diff --git a/src/docs/inbox-voice-sources.md b/src/docs/inbox-voice-sources.md new file mode 100644 index 0000000000..9947215654 --- /dev/null +++ b/src/docs/inbox-voice-sources.md @@ -0,0 +1,103 @@ +# Inbox and voice sources + +The **Inbox** (sidebar → Intelligence → 📥 Inbox) is where recordings and +dictations from voice sources land so you can turn them into work without +leaving the dashboard. It is source-agnostic: Plaud is the first source, +Wispr Flow (`type: wisprflow`) plugs in with the same model, and later sources +(file drop, webhook, Otter.ai, Fireflies, …) add nothing to the UI. + +Tracker RFC: *Voice recorders as first-class input sources (Inbox)* — +see the issue linked from the changelog entry. + +## Model + +- **Source** — a knowledge connector whose facts are recordings (`plaud`, + `wisprflow`). Connecting a source happens in **Knowledge → 🔌 Integrations**; + that card shows only the connection state and an **Open Inbox** button. +- **Item** — one recording, keyed `(source, item_id)` (for Plaud, the + `attr_plaud_file_id`). When the source is connected the Inbox lists + recordings directly from it, so an item may appear before the sync has + written its Knowledge fact; the Inbox only tracks what you did with it. + "📚 Fact" opens the fact in Knowledge once that fact exists. +- **Destination** — where an item goes: + - **→ RFC** (`spek_rfc`): the hive runs `spektacular voice --from stdin + --title --publish --repo <owner/repo>` on the transcript and + files one RFC issue plus native child issues with the hive handshake + marker. The hive's normal scanner then claims the children. If no + `spektacular` binary is on the host, the hive hands the same job to a + spoke agent as a kick prompt (state `delegated`): `inbox.spek.agent` if + set, otherwise the first enabled, unpaused agent by name. + - **→ dibs** (`dibs`): posts title, summary, transcript and the + `plaud://recordings/<id>` provenance to `POST {inbox.dibs.url}/api/ideas` + as a private draft idea (Bearer token from `inbox.dibs.token_env`). + - **Knowledge only** (`dismiss`): leaves the fact where it is and drops the + item from the unprocessed count. +- **State** per item per destination: `queued → running → filed | failed` + (or `delegated` for the agent fallback), with the RFC/idea URL and child + issue URLs. Stored in `connector-state/_inbox.json`. +- **Auto-routing rules** (⚙ Auto-routing in the Inbox, or `inbox.rules`): + `source?`, `title_prefix` or `tag`, `destination`, `repo?`. Rules run once + for each item that arrives after a connector sync; the first match wins; + items already in the Inbox are never re-routed. The same editor, with the + default RFC repository and dibs URL, is under Settings → Inbox. + +## Configuration + +```yaml +inbox: + sources: [plaud, wisprflow] # default: every configured voice connector + spek: + repo: hivecommons/hive # default RFC target + binary: spektacular # or an absolute path; empty = look on PATH + agent: "" # fallback agent; default: first enabled agent + dibs: + url: https://dibs.example.org + token_env: DIBS_TOKEN + rules: + - title_prefix: "RFC:" + destination: spek_rfc + - title_prefix: "Idea:" + destination: dibs +``` + +`inbox.rules`, `inbox.spek.repo` and `inbox.dibs.url` are editable from the +dashboard (`POST /api/inbox/settings`); the token is environment-only. + +## Hosted hives: signing in to Plaud + +Plaud's OAuth client only redirects to `http://localhost:8199/auth/callback`, +so a hive running in a cluster never receives the browser callback. The +Plaud card therefore offers **Finish sign-in by pasting**: + +1. Click **Connect Plaud**; open the authorization link and approve. +2. The browser lands on `localhost:8199/auth/callback?code=…` (a connection + error page is expected). Copy that URL and paste it into the card within + 10 minutes; the hive forwards it to the login session running in the pod. +3. Alternative: on a laptop run `npx -y @plaud-ai/mcp@latest install`, sign + in, and paste the contents of `~/.plaud/tokens-mcp.json`. + +Both go through `POST /api/knowledge/connectors/{name}/auth/complete`. +A browser-only redirect to the hive's public URL is not possible until Plaud +allows additional redirect URIs for its client. + +## Requirements on the hive image + +- Node.js ≥ 20 for the Plaud MCP server (the image ships Node 26). +- `spektacular` for → RFC: not in the image yet (tracked as a child of the + Inbox RFC); until then the agent fallback runs by default, or install it on + the host (`go install github.com/hivecommons/spektacular/cmd/spektacular@latest`). +- A GitHub token with issue-write on the target repo (the hive's app token + is injected as `GITHUB_TOKEN`). + +## API + +See [api-reference/knowledge.md](api-reference/knowledge.md) for +`GET /api/inbox`, `POST /api/inbox/route`, `POST /api/inbox/dismiss`, +`GET|POST /api/inbox/settings` and +`POST /api/knowledge/connectors/{name}/auth/complete`. + +`GET /api/inbox` always answers `200`: on a hive with no voice connector +configured it returns an empty `items` list and every source as +`configured: false`, and the Inbox panel shows its empty state with a +"Connect sources" button. A connector added under Knowledge → Integrations +starts syncing after the next hive restart. diff --git a/src/docs/integrations.md b/src/docs/integrations.md index f4479b2e3c..fed5bf8cad 100644 --- a/src/docs/integrations.md +++ b/src/docs/integrations.md @@ -94,6 +94,7 @@ per-backend confinement matrix. <tr><td><img src="assets/logos/podman.svg" height="40" alt="Podman logo"></td><td>Rootless Podman agent sandbox</td><td>Opt-in</td><td><a href="sandbox-isolation.md#current-wiring">sandbox-isolation.md</a></td></tr> <tr><td><img src="assets/logos/kubernetes.svg" height="40" alt="Kubernetes logo"></td><td>Kubernetes Job sandbox runtime</td><td>Opt-in</td><td><a href="sandbox-isolation.md#running-sandboxed-kicks-as-kubernetes-jobs">sandbox-isolation.md</a></td></tr> <tr><td><img src="assets/logos/nvidia.svg" height="40" alt="NVIDIA logo"></td><td>NVIDIA OpenShell sandbox runtime</td><td>In progress (opt-in, phase 1)</td><td><a href="sandbox-isolation.md#running-sandboxed-kicks-in-an-openshell-sandbox">sandbox-isolation.md</a> · <a href="https://github.com/hivecommons/hive/issues/17642">#17642</a></td></tr> +<tr><td><img src="assets/logos/cncf.svg" height="40" alt="CNCF logo"></td><td>Agent Substrate sandbox runtime (CNCF Sandbox)</td><td>In progress (opt-in, phase 2)</td><td><a href="sandbox-isolation.md#running-sandboxed-kicks-in-a-substrate-actor">sandbox-isolation.md</a> · <a href="https://github.com/hivecommons/hive/issues/17695">#17695</a></td></tr> <tr><td><img src="assets/logos/opensandbox-badge.svg" height="40" alt="OpenSandbox"></td><td>AAIF OpenSandbox (evaluation)</td><td>Planned</td><td><a href="https://github.com/hivecommons/hive/issues/14092">#14092</a></td></tr> </table> diff --git a/src/docs/knowledge-connectors.md b/src/docs/knowledge-connectors.md index 69f7c3b801..12dc561672 100644 --- a/src/docs/knowledge-connectors.md +++ b/src/docs/knowledge-connectors.md @@ -1,6 +1,6 @@ # Knowledge connectors -External knowledge systems (Confluence, Notion, Google Drive, SharePoint/OneDrive, git repositories, documents and GitHub wikis) are synced into the file vault by connectors configured under `knowledge.connectors` in `hive.yaml`. Each upstream page becomes one markdown fact, and agents read those facts like any other vault content. Operators manage connectors from the Knowledge card's **🔌 Integrations** modal or from **Settings → Knowledge → Connectors** in the dashboard. +External knowledge systems (Confluence, Notion, Google Drive, SharePoint/OneDrive, git repositories, documents, GitHub wikis, Plaud voice notes and Wispr Flow dictation) are synced into the file vault by connectors configured under `knowledge.connectors` in `hive.yaml`. Each upstream page becomes one markdown fact, and agents read those facts like any other vault content. Operators manage connectors from the Knowledge card's **🔌 Integrations** modal or from **Settings → Knowledge → Connectors** in the dashboard. For the shared configuration fields, the fact format and how connectors relate to `git_sources` and `documents`, see [knowledge-curator.md → Connectors](knowledge-curator.md#connectors-knowledgeconnectors). @@ -10,7 +10,7 @@ For the shared configuration fields, the fact format and how connectors relate t knowledge: connectors: - name: team-wiki # lowercase letters, digits, dashes - type: github-wiki # confluence | context7 | notion | git | document | github-wiki | google-drive | sharepoint + type: github-wiki # confluence | context7 | notion | git | document | github-wiki | google-drive | sharepoint | plaud | wisprflow enabled: true interval: 15m # minimum 1m; blank = 15m layer: project # personal | project | org | community @@ -218,6 +218,199 @@ knowledge: ends the content when any were dropped. See [Context7 content cap](knowledge-curator.md#context7-content-cap). +## Plaud Note Pro (voice notes, `type: plaud`) + +The Plaud connector turns the recordings on a [Plaud Note / Note Pro](https://plaud.ai) +into knowledge facts: one fact per recording with the recording time, +duration, Plaud's AI summary and the full speaker-labelled transcript. Agents +search and cite them like any other fact, so a voice memo recorded on the way +to work can inform the issue or PR an agent picks up an hour later. + +It talks to the **official, free Plaud MCP server** (`npx -y @plaud-ai/mcp`) +over stdio as an MCP client — there is no Zapier/Make/IFTTT hop and nothing to +pay beyond your existing Plaud plan. Plaud lists the MCP server as a Preview +feature on every tier including Starter; Plaud's docs note that preview +features may later move behind paid plans. + +```yaml +knowledge: + connectors: + - name: plaud + type: plaud + layer: personal + interval: 15m + scope: + command: npx -y @plaud-ai/mcp@latest # default; any stdio MCP command + since_days: "30" # how far back the first sync reaches + max_files: "200" # recordings fetched per sync + include_audio: "false" # add Plaud's 24h presigned MP3 link + query: "" # optional name filter (list_files query) + tags: "voice, ideas" # extra tags written as attr_tags + # home: /data/knowledge/connector-state/plaud/home # token store; default shown, or "inherit" +``` + +No `auth` block: Plaud signs you in with its own OAuth flow and keeps the token +in the MCP server's store (`$HOME/.plaud/tokens-mcp.json`). hive never sees, +stores or logs the token. By default the connector points the MCP server's +`HOME` at its own state directory under `/data/knowledge/connector-state/<name>/home` +so the token survives restarts; set `home: inherit` to use the hive process's +HOME instead. + +The MCP server is third-party code fetched at run time, so it does not inherit +the hive process environment. It is started with an allowlist only — `PATH`, +`HOME`, locale, `TMPDIR`, `TERM`, the `HTTP(S)_PROXY`/`NO_PROXY` and CA-bundle +variables, `NODE_*`/`npm_config_*` and `PLAUD_*` — and never the hub-injected +credentials (`DASHBOARD_AUTH_TOKEN`, `HIVE_TERMINAL_KEY`, `HIVE_UPSTREAM_PROXY_URL`, +provider API keys). Pin `scope.command` to a specific `@plaud-ai/mcp@<version>` +if you do not want a sync to pull a new upstream release unreviewed. + +### Setup + +1. The hive host (or container image) needs **Node.js ≥ 20** so `npx` can run + the Plaud MCP server. +2. Add the connector from **Knowledge → 🔌 Integrations → Plaud voice notes** + (or the YAML above) and restart the hive so the syncer picks it up. +3. On the Plaud card click **Connect Plaud**. The card shows the Plaud + authorization URL; open it, sign in and click **Authorize**. The OAuth + callback lands on `localhost:8199` of the machine running hive, so for a + remote hive forward that port first (`ssh -L 8199:localhost:8199 user@host`) + and open the URL from the forwarding machine. The login session stays open + for 10 minutes; click **Check** to confirm "connected as …". + - Hosted hive (no port forward): open **Finish sign-in by pasting** on the + card and paste either the `localhost:8199/auth/callback?…` URL the + browser landed on after approving, or the contents of + `~/.plaud/tokens-mcp.json` from a laptop where `npx -y + @plaud-ai/mcp@latest install` signed in. See + [Inbox and voice sources](inbox-voice-sources.md#hosted-hives-signing-in-to-plaud). +4. Click **Sync now**. New recordings are picked up every `interval` and + appear in the **📥 Inbox**, where you file them as a spektacular RFC, send + them to dibs, or keep them as knowledge only + ([Inbox and voice sources](inbox-voice-sources.md)). + +### Behaviour + +- Each sync calls `list_files` (from the newest recording seen minus a 24 h + overlap, or `since_days` back on the first run), then `get_note` and + `get_transcript` for each recording. A recording whose transcript and + summary are both still processing is skipped and retried next sync; the + status reports it as a partial sync. +- Facts carry `source: plaud`, `attr_plaud_file_id`, `attr_recorded_at`, + `attr_duration_ms` and `attr_device_serial`. Audio is never downloaded; + with `include_audio: true` the fact links Plaud's 24-hour presigned URL. +- The listing is incremental, so recordings deleted in Plaud are not + tombstoned automatically; remove the fact file or the connector. +- Sign-in state (`connected as`, pending authorization, last error) comes from + `GET /api/knowledge/connectors/{name}/auth`; see [API](#api). + +### Privacy and cost + +- Transcripts and summaries are produced in **Plaud's cloud** and fetched from + there; the hive stores them in the vault for the layer you choose (use + `personal` unless the whole hive should see them). Do not record material + that may not leave your organisation. +- **Local-only alternative:** Plaud's app can auto-sync notes to an Obsidian + vault or Drive/Dropbox folder, and exports TXT/Markdown/DOCX by hand. Point + the existing `git` connector at a Git-synced copy of that folder (see the + Obsidian entry in 🔌 Integrations), or import single files with a + `document` connector (`file_path`). No code, still $0, and no MCP sign-in. +- Everything in this section costs **$0** beyond your Plaud subscription. + +## Wispr Flow (dictation, `type: wisprflow`) + +[Wispr Flow](https://wisprflow.ai) is dictation into whatever field has focus: +hold the Flow key, speak, release, and polished text is pasted where the +cursor is. That gives two $0 paths into the hive. + +**Direct dictation — no connector.** Put the cursor in **Knowledge → + New +Fact** (title or body), the Import box, an agent prompt, or any other text +field in the dashboard and dictate. Nothing to configure. This is the +recommended way to get *new* thoughts in. + +**History connector.** The desktop app keeps every dictation in a local +SQLite file. The `wisprflow` connector reads it read-only and turns each +dictation (what you said, when, and into which app) into a fact, so ideas you +dictated into Slack or your editor last week are searchable by agents too. +There is no official export API for dictation history — Wispr's remote MCP +server covers Notetaker meetings only and states it cannot read dictation +history, and the Voice Interface API is invite-only and does speech-to-text, +not history — so the local file is the only free source. The schema is +community-documented (`History` table: `transcriptEntityId`, `asrText`, +`formattedText`, `editedText`, `timestamp`, `app`, `url`, `duration`, +`numWords`, `status`, `isArchived`) and can change with an app update. + +```yaml +knowledge: + connectors: + - name: wispr + type: wisprflow + layer: personal + interval: 15m + scope: + # path: /Users/me/Library/Application Support/Wispr Flow/flow.sqlite # default on macOS; %APPDATA%\Wispr Flow\flow.sqlite on Windows + # path: /data/knowledge/wispr-export # or a folder of exported .json/.ndjson/.csv/.md files + since_days: "30" # how far back the first sync reaches + max_items: "500" # dictations read per sync + min_words: "3" # skip "ok", "yes", … + group: dictation # dictation = one fact each; day = one fact per calendar day + apps: "" # optional allow-list matched against the app bundle id, e.g. "slack, obsidian, vscode" + include_raw: "false" # also keep the raw ASR text under "Raw transcript" + tags: "voice, ideas" # extra tags written as attr_tags + # sqlite3: sqlite3 # CLI used to read the file; ships with macOS and the hive image +``` + +No `auth` block and no sign-in: the data is already on disk. The connector +shells out to the `sqlite3` CLI with `-readonly -json` and selects text +columns only (the table also holds audio and screenshot BLOBs, which are never +read). Rows with `status = formatted` and `isArchived = 0` are used; +`editedText` wins over `formattedText` over `asrText`. + +### Setup + +1. **Same machine as the hive (laptop hive):** add the connector from + **Knowledge → 🔌 Integrations → Wispr Flow dictation** and leave `path` + empty. On macOS the hive process may need **Full Disk Access** (System + Settings → Privacy & Security) to read `~/Library/Application Support`. +2. **Hive on a cluster or another host:** sync the file or an export to a + path the hive can read and set `path` to it. Any of these work: + - copy `flow.sqlite` (or one of the app's timestamped `backups/*.sqlite`) + into the knowledge PVC, e.g. with a cron `kubectl cp`; + - run a community exporter such as `wispr export -o <dir>` + ([iamnikolie/wispr-cli](https://github.com/iamnikolie/wispr-cli), writes + `dictations/*.md`) or + [junxit/wispr-flow-exporter](https://github.com/junxit/wispr-flow-exporter) + (Markdown/JSON/NDJSON) and point `path` at the folder, or commit that + folder to a Git-synced vault and let the existing `git` connector pick + it up. +3. Click **Sync now**. New dictations are picked up every `interval`. + +Export folders are walked recursively; `.json` (array or +`{"dictations": [...]}`), `.ndjson`/`.jsonl`, `.csv` (header row) and `.md` +(one dictation per file; optional front matter `id:`, `timestamp:`, `app:`, +otherwise the date in the filename or the file mtime) are read, other files +are ignored. A file that fails to parse is reported as a partial sync and the +rest are still imported. + +### Behaviour + +- Incremental by `timestamp`: each sync re-reads from the newest dictation + seen minus a 2 h overlap (Flow updates a row in place as it finalises), or + `since_days` back on the first run. With `group: day` the current day's + fact is rebuilt whole. +- Dictation facts carry `source: wisprflow`, `source_id`, + `attr_recorded_at`, `attr_wispr_transcript_id`, `attr_dictated_at`, + `attr_app` (bundle id), `attr_num_words`, `attr_language`, and the + dictation date as the vault path (`Wispr Flow/2026-11-11`). The title is + the first ten words, and the formatted text is under `## Transcript`. + With `group: day`, the fact represents the entire day and its + `attr_recorded_at` is the newest dictation's time. +- The listing is incremental, so dictations you archive or delete in Flow + are not tombstoned automatically; remove the fact file or the connector. +- Everything here costs **$0** beyond your Wispr Flow plan; the free tier's + weekly word cap applies to dictating, not to reading history. Tip: add + your project vocabulary (`hive`, `dibs`, `spek`, `ACMM`, `crux`, repo + names) under Flow → Settings → Dictionary so it is spelled right the first + time. + ## Notion (`type: notion`) The Notion connector syncs pages and database rows through the public Notion diff --git a/src/docs/labels-and-control-signals/glossary-and-operations.md b/src/docs/labels-and-control-signals/glossary-and-operations.md index 145397af2c..e69f99058c 100644 --- a/src/docs/labels-and-control-signals/glossary-and-operations.md +++ b/src/docs/labels-and-control-signals/glossary-and-operations.md @@ -231,7 +231,7 @@ The dashboard renders every open issue and PR's stage as one unified tracker str ## Non-label equivalents - **Human comments and assignees** can acknowledge hive-filed directions for #5117; assignees also affect ranking. -- **Hidden comment markers** carry state that no label does. `<!-- hive:claim ... -->` records an issue claim, `<!-- hive-finding: HASH -->` dedupes a published finding, `<!-- hive-pr-overlap -->` marks an overlap notice, `<!-- hive-question-answer -->` starts the question auto-close clock, and `<!-- hive-shared-ci-<n> -->` is stamped on the single comment a fix lane leaves on a PR it defers to shared incident `#<n>` (`DEFER_TO_INCIDENT` from `bin/hive-baseline-check.sh`), so "every PR incident `#<n>` broke" stays greppable after the fix lands ([#10441](https://github.com/hivecommons/hive/issues/10441)). One marker per PR per incident; it is never edited or removed while the incident is open. While incident `#<n>` is open, the governor records it as `deferred_incident` on the PR's `ci-failing.json` row and the kick builders keep that PR out of every CI-FAILING / FIX-BEFORE-NEW repair list, naming it only as deferred; once the incident closes the PR is listed for repair again ([#10528](https://github.com/hivecommons/hive/issues/10528)). When the incident closes (not as "not planned"), the shared-CI rerun sweep re-runs the failed jobs of each still-open PR carrying that marker — the newest failed, timed-out or cancelled run per workflow on the PR head — using the hive's own credential, then stamps `<!-- hive-shared-ci-rerun-<n> -->` and `<!-- hive-shared-ci-lifted-<n> -->` on the PR so it is re-run at most once per incident; disable with `shared_ci_rerun.enabled: false` ([#12253](https://github.com/hivecommons/hive/issues/12253)). +- **Hidden comment markers** carry state that no label does. `<!-- hive:claim ... -->` records an issue claim, `<!-- hive-finding: HASH -->` dedupes a published finding, `<!-- hive-pr-overlap -->` marks the one overlap notice a PR gets (posted only when a sibling PR conflicts or shares a file, edited down to a "Resolved" line when that clears; [#17863](https://github.com/hivecommons/hive/issues/17863)), `<!-- hive-question-answer -->` starts the question auto-close clock, and `<!-- hive-shared-ci-<n> -->` is stamped on the single comment a fix lane leaves on a PR it defers to shared incident `#<n>` (`DEFER_TO_INCIDENT` from `bin/hive-baseline-check.sh`), so "every PR incident `#<n>` broke" stays greppable after the fix lands ([#10441](https://github.com/hivecommons/hive/issues/10441)). One marker per PR per incident; it is never edited or removed while the incident is open. While incident `#<n>` is open, the governor records it as `deferred_incident` on the PR's `ci-failing.json` row and the kick builders keep that PR out of every CI-FAILING / FIX-BEFORE-NEW repair list, naming it only as deferred; once the incident closes the PR is listed for repair again ([#10528](https://github.com/hivecommons/hive/issues/10528)). When the incident closes (not as "not planned"), the shared-CI rerun sweep re-runs the failed jobs of each still-open PR carrying that marker — the newest failed, timed-out or cancelled run per workflow on the PR head — using the hive's own credential, then stamps `<!-- hive-shared-ci-rerun-<n> -->` and `<!-- hive-shared-ci-lifted-<n> -->` on the PR so it is re-run at most once per incident; disable with `shared_ci_rerun.enabled: false` ([#12253](https://github.com/hivecommons/hive/issues/12253)). - **`<!-- hive-ci-external head=<sha> check=<encoded name> run=<check run id> -->`** is stamped by a fix lane on the one comment it leaves when `bin/hive-baseline-check.sh` returns `DEFER_EXTERNAL`: the repository syncs from an upstream project (its GitHub fork parent, or `--upstream`) and the same check is red on the upstream default branch, so no change here can fix it. The helper prints the exact string as `external_marker`. While every failing check run on the PR's current head carries a marker for that head and run, the governor records `deferred_external` on the PR's `ci-failing.json` row and the kick builders name the PR only as deferred under `DEFERRED AS FAILING UPSTREAM`; a push, a newly failing check or a re-run of the check (a new run id — e.g. it went green and failed again at the same head) makes the markers stale and the PR is listed for repair again ([#11609](https://github.com/hivecommons/hive/issues/11609)). - **`<!-- hive-ci-transient head=<sha> check=<encoded name> run=<check run id> -->`** is stamped by an agent on its triage comment when it diagnoses a red check on a PR as transient or infrastructure-related (a flaky job, a runner outage, a race against a cache the default branch was still publishing) rather than caused by the diff, and cannot re-run the check itself. The fields and pinning match `hive-ci-external`; `bin/hive-baseline-check.sh --json` with the PR number prints the exact string as `transient_marker` whenever the PR's newest run of the check is red, on any verdict, and the agent copies it verbatim. While every failing check run on the PR's current head carries a marker for that head and run, the PR's `transient_ci_triaged` is set. On a hive-authored PR the fix loop then re-runs the failed jobs once for that head (`rerun-failed-jobs`, gated as `ci_rerun`) and sends no FIX-BEFORE-NEW kick while the re-run is owed; a head still red after it is back in the ordinary fix loop ([projectbluefin/common#1647](https://github.com/projectbluefin/common/pull/1647)). Only markers in the hive's own comments count here, so a comment from anyone else cannot trigger the re-run or hold the fix loop. A hive that cannot name any of its own accounts falls back to every comment. A PR from `auto_merge.trusted_bot_authors` gets the same one re-run (on a bot PR `ci_rerun` needs the opt-in label unless `writeback.kinds.ci_rerun.unlabeled_min_level` is set), and the dependency-bot red-CI hand-off waits for it: it adds `needs-human` after its grace period only once the head is still red after the re-run, or the re-run re-ran nothing, and its comment says which (the re-run failed again, or it was not allowed, with the reason). For any other dependency bot the hand-off comes after the grace period with no re-run, and its comment says Hive's triage found the failure was not caused by the bump and asks for a re-run, instead of telling the owner to fix, pin or close the bump. A push or a re-run of the check (a new run id) makes the marker stale ([#12977](https://github.com/hivecommons/hive/issues/12977)). - **Paused repos** (`project.paused_repos`) stop whole-repo write/merge/enumeration paths without labels. diff --git a/src/docs/net-admin-requirement.md b/src/docs/net-admin-requirement.md index d0a9e06078..aea764ad62 100644 --- a/src/docs/net-admin-requirement.md +++ b/src/docs/net-admin-requirement.md @@ -127,6 +127,8 @@ match no traffic even mid-probe) and is unconditionally torn down: | `xt_REDIRECT` | the `REDIRECT` target that forces `:443` through the proxy | **FATAL, exit 77** | | `xt_owner` | owner-UID self-exemption (`-m owner`) | WARN only — optional by design; OpenShift/OVN runs without it and the mark exemption carries the proxy's egress | +Without `xt_owner` the mark is the only exemption, so every outbound `:443` dial the hive makes for itself must carry it. The proxy's upstream dials and the in-cluster Kubernetes API clients (self-upgrade, startup-probe backfill, node metrics, ingress probes, kubejob) share `crux/egressmark.Dialer`; an unmarked dial to `kubernetes.default.svc` would be redirected into the proxy and closed by its private-range forward guard (`EOF`), which is how IBM Cloud IKS spokes failed every self-upgrade with `reading startup probe before upgrade … EOF`. + When a required module is missing, the FATAL names it explicitly: ``` diff --git a/src/docs/network-requirements.md b/src/docs/network-requirements.md index 198496a7ed..ddb1af9e3e 100644 --- a/src/docs/network-requirements.md +++ b/src/docs/network-requirements.md @@ -97,6 +97,36 @@ slots on that cluster are provisioned with the env vars above: Existing hive Deployments are not reconciled; patch them once with `kubectl -n <hive-ns> set env deploy/hive -c hive HIVE_UPSTREAM_PROXY_URL=... HIVE_UPSTREAM_PROXY_HOSTS=...`. +### GitHub Enterprise behind a TLS reverse proxy (hostAliases) + +Some networks expose an enterprise GitHub not through a CONNECT forward proxy +but through a **TLS reverse proxy** pinned to a fixed IP: the proxy serves a +certificate valid for the GHE hostnames on :443 and forwards to GHE, while +`CONNECT` is rejected (e.g. openresty answering `405 Not Allowed`) and direct +egress to GHE times out. `upstream_proxy` cannot express that; the fix is a +Pod `hostAliases` entry that resolves the GHE hostnames to the proxy IP so TLS +SNI/verification and every path (API, git smart-HTTP, MITM) keep working. + +Hosted hubs declare it per cluster in `/data/saas/clusters.json`: + +```json +{ + "id": "corp-cluster", + "host_aliases": [ + { "ip": "129.34.104.21", "hostnames": ["github.ibm.com", "api.github.ibm.com"] } + ] +} +``` + +Entries are validated at load (the IP must parse, every entry needs at least +one hostname; hostnames are lowercased). An invalid `host_aliases` block is +logged and ignored — it never drops the cluster. New pool slots on the cluster +are provisioned with the matching `spec.template.spec.hostAliases`, and a +throttled reconcile sweep (at most 3 patches per 15 minutes, since each one +rolls that hive's pod) converges existing hives. Unrelated hand-applied +aliases for other IPs are preserved. Non-hub (standalone) deployments add the +same `hostAliases` block to their own pod spec. + ## Firewall recommendations - Public edge: expose only the dashboard/hub front door (3001 for Docker/hub, or your HTTPS Ingress/Route for Kubernetes). Keep 3002 and 7681 cluster/private unless you have an explicit operator access path. diff --git a/src/docs/notifications.md b/src/docs/notifications.md index 41f06b5b73..f7b37592bd 100644 --- a/src/docs/notifications.md +++ b/src/docs/notifications.md @@ -818,7 +818,32 @@ and exits non-zero instead of posting a possible duplicate. `.github/scripts/test-hive-status-card.sh` and `src/scripts/test-comment-merged-refs.sh` pin this behaviour. +### Per-PR workflow comments post only on a finding + +Three workflow comments used to land on every PR regardless of outcome, and +after the runtime's own reducers they were ~70% of the comments on a merged +PR ([#17863](https://github.com/hivecommons/hive/issues/17863)). Each now +posts only when it has something to say; the full report, including the +clean case, still goes to the job's step summary: + +| Workflow / script | Comment | Behaviour | +| --- | --- | --- | +| Open PR Overlap (`.github/workflows/pr-overlap.yml`, `.github/scripts/pr-overlap.sh`) | `<!-- hive-pr-overlap -->` table of conflicting / same-file open PRs | **Posts** only when at least one sibling PR conflicts or shares a file; **edits** the one marker comment in place on later pushes; when a later run finds nothing and a comment exists, **edits it** to a one-line "✅ Resolved" note; a PR that never had an overlap gets no comment. The `conflicts-with-open-pr` label is unchanged. | +| Mutation (advisory) (`.github/workflows/mutation-advisory.yml`, `src/scripts/mutation-report.sh post`) | `<!-- hive-mutation -->` surviving / uncovered mutant report | **Posts** only for a non-`pass` verdict (`lived`, `uncovered`, `misrouted`, `error`); **edits** the one marker comment on later pushes; a later `pass` **edits** the existing comment to a one-line "✅ Resolved" note; a PR whose every run passes gets no comment. | +| Feedback Wanted (`.github/workflows/feedback.yml` → `hivecommons/infra` `reusable-feedback.yml`, gated by `src/scripts/feedback-gate.sh`) | "Thank you for your contribution! Your PR has been merged." | **Posts** only for a person's first merged PR in the repo. Skipped for `[bot]` / App authors, hive-managed head branches (`scanner/*`, `hive/*`), PR bodies carrying a `<!-- hive:` marker (hive-agent PRs on any branch, e.g. `fix-*`), and any author with another merged PR. Knobs are the workflow's `env:` (`FEEDBACK_SKIP_BOTS`, `FEEDBACK_SKIP_BRANCHES`, `FEEDBACK_SKIP_HIVE_ORIGIN`, `FEEDBACK_SKIP_LOGINS`, `FEEDBACK_FIRST_TIME_ONLY`); a failed merged-PR lookup skips the comment rather than guessing. | + +`.github/scripts/test-pr-overlap.sh`, `src/scripts/test-mutation-report.sh` +and `src/scripts/test-feedback-gate.sh` pin these rules. + **Outside Hive's control.** Comments from external bots, such as Prow, tide and kubestellar-prow, are posted by their own services and cannot be edited, folded or skipped from this repository. Hive does not try to suppress them; -configure those bots in the project that runs them. +configure those bots in the project that runs them. For hivecommons repos +that configuration is `prow/plugins.yaml` and `prow/config.yaml` in +`kubestellar/infra` (org-wide `approve` with `require_self_approval: true` +and `lgtm` with `review_acts_as_lgtm: true`; tide requires both `lgtm` and +`approved`). What Hive does control is its own relay: a dashboard approval +on a Prow-managed PR the hive account opened posts `/approve` only, because +a `/lgtm` from the PR's own author draws Prow's "you cannot LGTM your own +PR" every time (`PullRequestApprovalEvent.LGTMSkipped`, #17863); no hive +lane reads the relayed `/lgtm`. diff --git a/src/docs/operator-reference/runtime-and-quota.md b/src/docs/operator-reference/runtime-and-quota.md index 13a2881ca8..896126e5c6 100644 --- a/src/docs/operator-reference/runtime-and-quota.md +++ b/src/docs/operator-reference/runtime-and-quota.md @@ -198,6 +198,57 @@ loops, so the proxy enforces two safeguards: once. Bodies are stored gzip-compressed, so the cap is several times its size in raw JSON. +### Hub bearer validation and exhausted user tokens + +The hub validates a `Bearer` token it has not seen before with one +`GET /user` **using that token**, then caches the login for 5 minutes. A token +GitHub rejects (401, or 403/429 with `X-RateLimit-Remaining: 0`) is now also +remembered: for 60 s by default, or until the token's `X-RateLimit-Reset` +(capped at 15 minutes) when it is rate-limited. Without that negative entry a +client retrying with an exhausted token made the hub re-probe GitHub on every +request, charging the token owner's own core bucket while it was already at +zero. The probe is recorded in `/api/gh-rate-limits` under caller +`agent:hub-bearer-validate`, so a flood of 401s from one dead token is visible +on the hub admin page; being a pass-through credential it never updates the +hub's own core-bucket snapshot. + +### Keep personal tokens out of cluster secrets + +GitHub's 5,000 req/h core limit is **per user**, shared by every PAT and +OAuth token (`gho_…`, `ghp_…`, `github_pat_…`) that user owns, from every +host. A `gh auth token` pasted into a cluster secret therefore shares its +owner's bucket with their laptop `gh`, the hub's bearer validation of that +user, and any other service holding a copy. The hub itself authenticates as +the GitHub App (`HIVE_HUB_GITHUB_APP_ID` / `HIVE_HUB_GITHUB_INSTALLATION_ID` / +`HIVE_HUB_GITHUB_APP_KEY_FILE`, see [env-vars.md](../env-vars.md)) and uses +`HIVE_HUB_GITHUB_TOKEN` only as a fallback, logging a startup `WARN` naming the +identity it ended up with. Audit the rest of the fleet the same way: + +1. **Find copies.** For every cluster context, compare the first 8 characters + of each GitHub-looking secret value and `/data/gh-user-token` file against + the owner's token (`kubectl get secret -A -o json`, decode, never print the + full value). Known holders in a hive deployment are the Actions Runner + Controller (`gha-runner-scale-set`) `githubConfigSecret`, its generated + `*-listener-config` secret, and any sidecar that scrapes GitHub (dibs). +2. **Move ARC to App auth.** The runner scale-set listener long-polls the + Actions broker and acknowledges every message; with a PAT those calls are + charged to the PAT owner (~3,000 req/h at 350 ephemeral runners/h). Grant + the GitHub App repository **Administration: read & write** (or organization + **Self-hosted runners: read & write** for an org-level runner group), + then replace the `githubConfigSecret` keys `github_token` with + `github_app_id`, `github_app_installation_id` and + `github_app_private_key` (the same PEM mounted on the hub as + `hive-hub-github-app-key`), and delete the listener pod so the controller + regenerates its `*-listener-config` from the new secret. App traffic lands + in the installation's own 15,000 req/h bucket. +3. **Remove the PAT from the hub.** With the three App variables set, unset + `HIVE_HUB_GITHUB_TOKEN` on `deploy/hive-hub`; the startup log then reads + `hub GitHub API identity selected mode=app` instead of the `WARN`, and + `GET /api/gh-rate-limits` reports the App installation's `core.limit`. +4. **Rotate.** Once no cluster holds it, revoke the leaked user token + (`gh auth refresh` mints a new `gho_`), so stale copies fail closed rather + than silently sharing the bucket again. + The proxy returns JSON shaped for the GitHub CLI. For the agent's own hourly cap: diff --git a/src/docs/releases.md b/src/docs/releases.md index 0dc22efe02..704bc9de42 100644 --- a/src/docs/releases.md +++ b/src/docs/releases.md @@ -212,13 +212,18 @@ separate, deliberate policy described in silently couples to it, on purpose — the operator explicitly asked for these to stay decoupled. -Hub-managed hives on the `candidate` channel are canaries: by default the hub -does not hold them behind the daily/weekly auto-upgrade cadence. Once the -candidate image for a newer commit is published, the hub may arm the hive on the -next poll as long as the existing idle, health and wave gates allow it. Stable -channel hives keep the slower soak/cadence rules. Operators who deliberately -want candidate hives to obey their stored cadence can set -`HIVE_CANDIDATE_RESPECT_AUTO_UPGRADE_SCHEDULE=true` on the hub. +Hub-managed hives on the `candidate` channel follow their stored auto-upgrade +cadence (`auto_upgrade_mode`: `daily` by default, `weekly`, or `instant`) just +like stable hives (#11816). A daily candidate hive therefore picks up the newest +published candidate image once a day, in the first poll at or after 1pm ET; the +hub reports the hold as `upgradeWhyNotNow: "before the daily window"` and the +spoke's `/api/version` shows `state: behind` until then. To track every +published candidate image, set the hive's mode to `instant` +(`PUT /api/saas/hives/{id}/auto-upgrade` with +`{"auto_upgrade": true, "auto_upgrade_mode": "instant"}`, or the Manage → +Auto-upgrade control on the hub). Setting +`HIVE_CANDIDATE_RESPECT_AUTO_UPGRADE_SCHEDULE=false` on the hub restores the +old behaviour where every candidate hive bypasses its cadence. The hub itself does **not** follow the hosted-hive daily/weekly cadence: when hub auto-upgrade is enabled and not paused, the hub rolls itself **instantly** diff --git a/src/docs/running-at-level-6.md b/src/docs/running-at-level-6.md index 6c719e8473..a1825be9e3 100644 --- a/src/docs/running-at-level-6.md +++ b/src/docs/running-at-level-6.md @@ -152,7 +152,7 @@ Config keys to cross-check: `project.issue_filter.require_labels`, `project.issu | `ci-flake` | CI/Hive scripts | Flake tracking, not an L6 hold itself. | Fix/quarantine the flake while keeping required checks meaningful. | | `user-feedback` | Dashboard/hub | Feedback provenance. | Triage reporter input; apply trust/hold policy as needed. | | `dco` / `no-dco` (Prow's `dco-signoff: yes|no` coexist on Prow-gated repos; the hive never adds or removes them) | Hive stage pass (commit check) | Merge-eligibility display on every managed repo. Where DCO is required (detected from required/head checks or configured via `governor.labels.stage.dco.required`), `no-dco` routes by author kind: `070` (hive/self-authored), `071` (human), or `072` (clanker); the 072 nudge includes the rebase command. On ungated repos it is informational only: the lane's stage is unchanged and no DCO rebase is requested. See [DCO routing](labels-and-control-signals/stage-labels.md#dco--no-dco--the-hive-owned-sign-off-signal). | Fix commit sign-offs (`git rebase --signoff`) rather than hand-editing labels. | -| `conflicts-with-open-pr` | PR-overlap workflow | Overlap signal. | Resolve or close conflicting work before merging; same-file overlap alone may be harmless. Two open PRs that both `Fixes` one issue are resolved by the duplicate-fix sweep without a decision: the earlier green/pending PR stays, the later hive PR is closed as `duplicate-of:<keeper>`, a person's later PR is commented on and left open ([#14799](https://github.com/hivecommons/hive/issues/14799)). | +| `conflicts-with-open-pr` | PR-overlap workflow | Overlap signal (the companion `<!-- hive-pr-overlap -->` comment appears only when there is a conflict or same-file overlap). | Resolve or close conflicting work before merging; same-file overlap alone may be harmless. Two open PRs that both `Fixes` one issue are resolved by the duplicate-fix sweep without a decision: the earlier green/pending PR stays, the later hive PR is closed as `duplicate-of:<keeper>`, a person's later PR is commented on and left open ([#14799](https://github.com/hivecommons/hive/issues/14799)). | | `has-open-pr` | Pre-dispatch guard | An open PR already links the issue with a closing keyword; the hive skipped the claim. | Nothing — the label is a tracker signal, not a park; see `scanner.skip_issues_with_open_pr` in [operator-reference.md](operator-reference.md#open-pr-dispatch-guard-and-duplicate-fix-sweep-scanner). | | `no-changelog` | Dependabot/maintainer | Repo changelog exception, not a Hive safety gate. | Apply only under repo policy. | | `human-only` remainder text (not a default label) | Agent/reviewer | Task-list sweep can park remaining issue work as `needs-human`. | Treat as a handoff; add PR `hold` if merging must stop. | diff --git a/src/docs/sandbox-isolation.md b/src/docs/sandbox-isolation.md index f3eaa320c1..ec26e05e87 100644 --- a/src/docs/sandbox-isolation.md +++ b/src/docs/sandbox-isolation.md @@ -280,16 +280,59 @@ The gateway, workspace, token, TLS and claim fields are cluster facts and fall b **The credential boundary.** Same rule as the Job runtime: environment variables from the allowlist are copied in with the same credential-name filter, hive's brokered GitHub token never enters the sandbox, and hive pushes and opens the PR from the returned workspace. There is **no** Secret-reference path into the sandbox in phase 1, so model credentials have to arrive through an allowlisted non-credential variable pointing at a routable gateway named in `allow_hosts`. The Cline backend, which needs a provider file or Secret, is refused on this runtime. -**Lifecycle.** The sandbox is named `hive-kick-<agent>-<unix>` and labelled `app.kubernetes.io/managed-by=hive-openshell`. Hive waits for it to become ready, runs the kick command with the sanitized environment in `/workspace`, collects stdout, stderr and the exit code (capped at 16 MiB), and deletes the sandbox — on success, on failure, and on the kick timeout, in which case the kick ends with exit code `-1` and `context deadline exceeded`, exactly like a timed-out Job. Cleanup runs on a fresh, bounded context when the kick's own has expired, so a finished sandbox is not left behind because the deadline fired during log collection. +**Lifecycle.** The sandbox is named `hk-<agent>-<unix, base36>` (at most 19 characters, the gateway's limit for DNS-routable names) and labelled `app.kubernetes.io/managed-by=hive-openshell`. Hive waits for it to become ready, runs the kick command with the sanitized environment in `/workspace`, collects stdout, stderr and the exit code (capped at 16 MiB), and deletes the sandbox — on success, on failure, and on the kick timeout, in which case the kick ends with exit code `-1` and `context deadline exceeded`, exactly like a timed-out Job. Cleanup runs on a fresh, bounded context when the kick's own has expired, so a finished sandbox is not left behind because the deadline fired during log collection. **What hive checks, and what it cannot.** Hive warns at boot and on reload (`config.AgentSandboxGateWarnings`) when an openshell-runtime agent resolves no `gateway_url` or no `workspace_claim`, and refuses the kick with the same message before the agent is marked running. It also preflights each distinct gateway (`Health().Check` and `Config().GetGateway`, read-only, cached for five minutes, bounded to 30 s) and adds a warning when the gateway is unreachable or unhealthy, `allow_driver_config` is not enabled, or telemetry is on. It cannot see the node kernel, the CNI or the claim's labels from inside its pod; those remain the operator prerequisites in [deploy/openshell.md](../deploy/openshell.md). -**The terminal pane.** An openshell-runtime agent has no tmux session either, so its ▶ terminal opens the same read-only `/terminal/job/<agent>` viewer as a job-runtime agent, behind the same handoff and role check ([#17645](https://github.com/hivecommons/hive/issues/17645)). While a kick runs it follows the newest live `hive-kick-<agent>-<unix>` sandbox's gateway log (`GetLogs`, polled at up to the gateway's 2,000-line buffer per read, scrubbed of token-shaped values like the pod log); between kicks, once the launcher has deleted the sandbox, or when the gateway cannot be listed, it shows the last persisted transcript exactly as the job viewer does. The live view is weaker than the job pod log, and the pane says so: the gateway log carries the sandbox's supervisor, policy (OCSF) and tracing events — the network and filesystem decisions made around the kick — not the kick's stdout, because the kick runs as an exec session and [exec output reaches neither the gateway log nor the container log](https://docs.nvidia.com/openshell/observability/accessing-logs). Likewise the exit code the gateway reports on the sandbox is its main process's, not the kick's; the launcher takes the kick's exit code from the exec result and deletes the sandbox at once. So the pane ends by saying the kick has finished and that its output and exit code are in the transcript, which it shows on the next open. The dashboard reaches the gateway with the agent's own `gateway_url`, `namespace`, `token_env` and `insecure_tls`. +**The terminal pane.** An openshell-runtime agent has no tmux session either, so its ▶ terminal opens the same read-only `/terminal/job/<agent>` viewer as a job-runtime agent, behind the same handoff and role check ([#17645](https://github.com/hivecommons/hive/issues/17645)). While a kick runs it follows the newest live `hk-<agent>-<unix, base36>` sandbox's gateway log (`GetLogs`, polled at up to the gateway's 2,000-line buffer per read, scrubbed of token-shaped values like the pod log); between kicks, once the launcher has deleted the sandbox, or when the gateway cannot be listed, it shows the last persisted transcript exactly as the job viewer does. The live view is weaker than the job pod log, and the pane says so: the gateway log carries the sandbox's supervisor, policy (OCSF) and tracing events — the network and filesystem decisions made around the kick — not the kick's stdout, because the kick runs as an exec session and [exec output reaches neither the gateway log nor the container log](https://docs.nvidia.com/openshell/observability/accessing-logs). Likewise the exit code the gateway reports on the sandbox is its main process's, not the kick's; the launcher takes the kick's exit code from the exec result and deletes the sandbox at once. So the pane ends by saying the kick has finished and that its output and exit code are in the transcript, which it shows on the next open. The dashboard reaches the gateway with the agent's own `gateway_url`, `namespace`, `token_env` and `insecure_tls`. **How to verify.** With a gateway reachable and the claim attachable, run the env-gated integration test from a pod that has the claim mounted: `OPENSHELL_GATEWAY_URL=… OPENSHELL_WORKSPACE_CLAIM=hive-data OPENSHELL_IMAGE=<digest-pinned> go test ./crux/sandbox -run TestOpenShellIntegration -v`. It writes a file through `/workspace`, reads it back on the hive side, and logs the round-trip latency. Then run a kick and confirm in the sandbox log that a `POST` to `api.github.com` is denied while `git fetch` succeeds — that denial is the whole point of the runtime. **Limitations (phase 1).** Kubernetes driver only (no Podman-driver path for standalone hosts yet); no Secret references into the sandbox; `policy_profile` is advisory; kernel and claim-label prerequisites are not preflighted; latency and image size are not yet measured ([#17643](https://github.com/hivecommons/hive/issues/17643)). +## Running sandboxed kicks in a Substrate actor + +`sandbox.runtime: substrate` runs an agent's sandboxed kicks as an [Agent Substrate](https://github.com/agent-substrate/substrate) **actor** ([ADR-0026 phase 2](adr/0026-sandbox-runtime-selection.md#implementation-notes-phase-2-agent-substrate), epic [#17695](https://github.com/hivecommons/hive/issues/17695)). Substrate is a CNCF Sandbox project donated by Google: a `WorkerPool` of pods runs workloads under gVisor (`runsc`) or in a KVM microVM, every actor gets a SPIFFE identity, and all egress leaves through `atenet-router` under an `EgressPolicy`. It is the fourth runtime and, like OpenShell, opt-in; nothing changes for a hive that does not select it. + +Everything above about the kick path still holds; only step 2 changes. The launcher is `sandbox.SubstrateLauncher`, which talks to Substrate's gRPC **Control API** (`ate.control.v1`) with a trimmed, wire-compatible client generated in-repo (`crux/sandbox/substrateapi`; the upstream Go module needs a newer Go than hive builds with). + +```yaml +agent_sandbox: + runtime: substrate + substrate: + endpoint: api.ate-system.svc:443 # Control API, host:port or https:// + atespace: hive # Substrate's tenancy unit, default "default" + token_env: ATE_TOKEN # bearer token env var (or token_file for a projected SA token) + ca_file: /var/run/ate/ca.crt # Control API CA; insecure_tls: true only for a lab + router_url: https://atenet-router.ate-system.svc:8444 + router_client_bundle: /var/run/ate/credential-bundle.pem # hive's own mTLS client identity + router_ca_file: /var/run/ate/router-ca.crt + image: ghcr.io/hivecommons/crux-sandbox@sha256:… # digest-pinned; tags are refused + cpu_millicores: 2000 + memory_bytes: 4294967296 + worker_selector: {ate.dev/sandboxClass: gvisor} # pick the pool (gvisor | microvm) + snapshot_on_idle: false + allow_hosts: [models.example.com, hub.example.com:8443] + timeout_s: 1800 +agents: + reviewer: + sandbox: + substrate: + snapshot_on_idle: true # per-agent overrides merge onto the global block +``` + +**One actor per kick.** Hive creates an `ActorTemplate` (one container running the bridge command, the workspace as a `durableDir`, the operator's limits and `worker_selector`) and an `Actor` from it, both named `hive-kick-<agent>-<unix>`, attaches the egress policy, waits for the actor to report `RUNNING`, and then talks to a small HTTP **bridge** inside the actor (`crux substrate-bridge`, overridable with `bridge_command`, port `bridge_port`) through the router's mTLS `CONNECT` tunnel: it pushes the workspace as a tar, runs the kick with the sanitized environment, and pulls the workspace back. The hive-side workspace remains the source of truth and feeds the push broker exactly as for Podman. Substrate has no exec API and hive shares no PVC with the worker nodes, which is why the workspace is copied rather than mounted; the copy is bounded by the kick timeout. + +**The egress policy.** Hive derives an `atenet-egress` policy from the hosts its proxy brokers for a Podman sandbox — `github.com` and `api.github.com` on 443 — plus `allow_hosts` as `host` or `host:port`, all as TLS passthrough; URLs, userinfo and wildcards are rejected at validation, before any actor is created. Everything else, including all UDP and raw IP connects, is denied. Substrate's policy language has no per-method read-only mode, so `api.github.com` is reachable; the actor holds no token (`SanitizedEnv`) and hive performs every GitHub write itself after the kick. `network_mode: none` drops every rule. Actors run credential-free: there is no `HTTPS_PROXY` or MITM broker inside the actor and no credential enters it, and hive brokers pushes after the kick from the returned workspace (decision on [#17695](https://github.com/hivecommons/hive/issues/17695), [ADR-0026](adr/0026-sandbox-runtime-selection.md)). Substrate's own credential injection is not used; SVID-authenticated broker delegation over the atenet mTLS channel is a future phase 3. + +**Lifecycle.** Success, failure and the kick timeout all delete the actor and then its template, on a fresh bounded context when the kick's own has expired, with exit code `-1` and `context deadline exceeded` on timeout, like a timed-out Job. With `snapshot_on_idle: true` the actor is **suspended** (snapshotted to Substrate's object storage) instead of deleted; the template is still deleted and the snapshot is not yet resumed on the next kick ([#17697](https://github.com/hivecommons/hive/issues/17697)). At most one suspended actor is retained per agent per hive process: the previous one is deleted after the next successful suspend (skipped with a warning if the agent's `atespace` changed in between), and actors left by an earlier hive process are not reclaimed because the Control API has no `ListActors`. `policy_profile` is passed to the actor as `HIVE_POLICY_PROFILE` for in-actor tooling; Substrate has no annotations. + +**What hive checks, and what it cannot.** At boot and on reload (`config.AgentSandboxGateWarnings`) a substrate-runtime agent is warned about, and its kick refused, when `endpoint` or `router_url` is missing or `image` is not digest-pinned. Hive also preflights each distinct Control API (read-only `ListWorkers`, cached for five minutes, bounded to 30 s) and warns when it is unreachable, has no `ACTIVE` worker matching `worker_selector`, or its workers advertise no sandbox class. It cannot see the CRDs, the operator, the pod-certificate feature gates or `/dev/kvm` from inside its pod; those are the operator prerequisites in [deploy/substrate.md](../deploy/substrate.md) and the prover in [#17698](https://github.com/hivecommons/hive/issues/17698). + +**The terminal pane.** A substrate-runtime agent has no tmux session either, so its ▶ terminal opens the same read-only `/terminal/job/<agent>` viewer behind the same handoff and role check ([#17777](https://github.com/hivecommons/hive/issues/17777)). The Control API has no `ListActors` and Substrate has no logs RPC, so the launcher — which runs in the same hive process as the dashboard — records each kick for the pane: the actor's `<atespace>/<name>`, every state it is seen in (`RESUMING`, `RUNNING`, then `SUSPENDED` with `snapshot_on_idle` or `DELETING`), and the worker's SPIFFE id from the actor's worker assignment. The bridge's `POST /run` streams the kick's stdout and stderr when asked (`"stream": true`, newline-delimited `RunResponse` frames; an older bridge answers with one complete response and still works), and the pane follows that output live, scrubbed of token-shaped values like the pod log, with each actor state change as a status line. Unlike OpenShell this is the kick's own output, and the pane ends with the kick's exit code and whether the actor was suspended or deleted. Between kicks it names how the last actor ended above the last persisted transcript; after a hive restart, before the agent's next kick, it shows the transcript alone. + +**Limitations (phase 2).** Workspace copied, not shared; `crux substrate-bridge` entrypoint and a published sandbox image land with the prototype ([#17697](https://github.com/hivecommons/hive/issues/17697)); no resume of suspended actors; `api.github.com` not read-only at the policy layer; latency and image size not yet measured. + ## Health and metrics for sandboxed agents A sandboxed agent (Podman or Job runtime) has no tmux session, no in-pod process and no `pluk` log between kicks: the manager parks it in the `idle` state and each kick runs to completion in a container or Job. The dashboard therefore must not read "not running" as "session down" for these agents, and the token collector cannot find their CLI session files because the CLI's home directory lives in the sandbox, not in the hive pod. @@ -319,7 +362,7 @@ On-demand agents, paused agents and `pause`/`continuous` cadences are never over - The Job runtime is a first slice: one Job per kick, workspace shared through a PVC, no per-kick Secret minting. A clone-inside-the-Job transport for clusters without ReadWriteMany storage is future work. - Live Podman execution is covered by skip-when-absent tests; CI still needs a rootless-Podman runner lane for always-on integration coverage. - Sandboxed inference depends on an operator-provided restricted network/proxy policy. `none` is stronger but not yet usable for all model backends. -- A stronger runtime than plain containers is under evaluation: [ADR-0026](adr/0026-sandbox-runtime-selection.md) (Proposed) compares NVIDIA OpenShell and OpenSandbox against this contract. +- Stronger runtimes than plain containers ship opt-in: [ADR-0026](adr/0026-sandbox-runtime-selection.md) (Accepted) covers NVIDIA OpenShell (phase 1) and Agent Substrate (phase 2); both still need their prototype measurements and cluster preflights. ## Agent guardrails (defense in depth) diff --git a/src/docs/scale-envelope.md b/src/docs/scale-envelope.md index bce071a20c..3b32dda14e 100644 --- a/src/docs/scale-envelope.md +++ b/src/docs/scale-envelope.md @@ -34,7 +34,7 @@ ones to: 2. **Truncation must preserve the prioritization signal, not cut an arbitrary slice.** This holds today and is easy to assume backwards: - Issues are sorted by actionable rank tier, then **oldest first** within each tier before any cap is applied - (`RankActionableIssues`, `src/pkg/github/client.go:5512-5521`, descending `AgeMinutes` within each tier). + (`RankActionableIssues`, `src/pkg/github/client.go:5535-5545`, descending `AgeMinutes` within each tier). - PRs are sorted by **review class then oldest-first within class** (`SortPullRequestsForReview`, `src/pkg/github/review_priority.go:334`; fixes → refactors/docs → tests). diff --git a/src/docs/troubleshooting.md b/src/docs/troubleshooting.md index 9197710220..6bde57e43e 100644 --- a/src/docs/troubleshooting.md +++ b/src/docs/troubleshooting.md @@ -822,7 +822,9 @@ when self-reporting is enabled, as a `[selfreport]` issue. mutation journal (`/data/convergence/mutation/journal.json` in the pod). Check whether the effect exists on GitHub; the owning call site reconciles it on its next pass, or the op resolves once the request behind it is removed. A watcher retrying - a doomed request keeps the op pinned, so fix the request first. + a doomed request keeps the op pinned, so fix the request first. An + `issue_create` op still refused after 1h is expired to NotApplied by the + boundary itself and the report clears. - **`unclassified_error_repeated`** — any WARN/ERROR line repeated five times in ten minutes. The report's message is the normalised line; grep the log for it (`kubectl -n hive logs deploy/hive | grep -F "<message>"`) to find diff --git a/src/internal/testutil/dashboard_import_allowlist.txt b/src/internal/testutil/dashboard_import_allowlist.txt index 0ea7bc5740..23afcfe6da 100644 --- a/src/internal/testutil/dashboard_import_allowlist.txt +++ b/src/internal/testutil/dashboard_import_allowlist.txt @@ -22,10 +22,12 @@ # search: GET /api/repos/{owner}/{repo}/health serves the report-only repo health score the search indexer computes (search.Health), wired through a setter so the dashboard never builds or walks the index (hivecommons/hive#14638). # search: GET /api/search reads the portfolio search Backend the indexer publishes; pkg/search is a leaf (no dashboard deps) (hivecommons/hive#14637). # workflowmodel: /api/workflows handlers compile lifecycle models (Compile, Profile diagnostics) and the tracker emits per-repo workflow columns; the store and bindings already route through Dependencies (hivecommons/hive#17620 P2). +# egressmark: deployment_upgrade.go dials the in-cluster Kubernetes API for the hive's own rollout; the dial must carry the forced-egress SO_MARK or nodes without xt_owner redirect it into the MITM proxy (hivecommons/hive#17817). github.com/hivecommons/hive/crux github.com/hivecommons/hive/crux/agent github.com/hivecommons/hive/crux/agentparse github.com/hivecommons/hive/crux/claude +github.com/hivecommons/hive/crux/egressmark github.com/hivecommons/hive/crux/kubejob github.com/hivecommons/hive/crux/logread github.com/hivecommons/hive/crux/logscrub @@ -67,6 +69,9 @@ github.com/hivecommons/hive/pkg/housestyle github.com/hivecommons/hive/pkg/hub github.com/hivecommons/hive/pkg/hubactivity github.com/hivecommons/hive/pkg/hubbackup +# pkg/inbox: Inbox triage API (hivecommons/hive#17876) — dashboard routes +# /api/inbox* wrap the inbox service; no existing seam covers it. +github.com/hivecommons/hive/pkg/inbox github.com/hivecommons/hive/pkg/inferencehealth github.com/hivecommons/hive/pkg/ioscan github.com/hivecommons/hive/pkg/knowledge diff --git a/src/pkg/appkey/appkey.go b/src/pkg/appkey/appkey.go index 1e8772937f..9a57a80fc0 100644 --- a/src/pkg/appkey/appkey.go +++ b/src/pkg/appkey/appkey.go @@ -20,6 +20,8 @@ import ( "path/filepath" "strconv" "strings" + "sync" + "time" "github.com/hivecommons/hive/pkg/config" ) @@ -62,6 +64,17 @@ type Resolver struct { ProvisionedKeyPath string // FileMode is applied to keys this Resolver writes. FileMode os.FileMode + + fpMu sync.Mutex + fpCache map[string]heldFingerprint +} + +// heldFingerprint memoizes a key file's fingerprint against the size and mtime +// it was computed from, so an unchanged file is never re-read. +type heldFingerprint struct { + size int64 + mtime time.Time + fp string } type tempKeyFile interface { @@ -77,8 +90,9 @@ var ( createTempKey = func(dir, pattern string) (tempKeyFile, error) { return os.CreateTemp(dir, pattern) } - removeFile = os.Remove - renameFile = os.Rename + fingerprintFromFile = config.AppKeyFingerprintFromFile + removeFile = os.Remove + renameFile = os.Rename ) // Default returns a Resolver using the production spoke layout. @@ -125,6 +139,10 @@ func (r *Resolver) HeldFingerprints() map[string]string { return nil } var held map[string]string + r.fpMu.Lock() + defer r.fpMu.Unlock() + seen := make(map[string]heldFingerprint) + defer func() { r.fpCache = seen }() for _, e := range entries { if e.IsDir() { continue @@ -138,10 +156,21 @@ func (r *Resolver) HeldFingerprints() map[string]string { if convErr != nil || id <= 0 { continue } - fp, fpErr := config.AppKeyFingerprintFromFile(filepath.Join(r.DataDir, name)) - if fpErr != nil || fp == "" { + var fp string + info, infoErr := e.Info() + if infoErr != nil { continue } + if c, ok := r.fpCache[name]; ok && c.size == info.Size() && c.mtime.Equal(info.ModTime()) { + fp = c.fp + } else { + var fpErr error + fp, fpErr = fingerprintFromFile(filepath.Join(r.DataDir, name)) + if fpErr != nil || fp == "" { + continue + } + } + seen[name] = heldFingerprint{size: info.Size(), mtime: info.ModTime(), fp: fp} if held == nil { held = make(map[string]string) } diff --git a/src/pkg/appkey/write_error_test.go b/src/pkg/appkey/write_error_test.go index 084a2ac422..0e7c91a1fa 100644 --- a/src/pkg/appkey/write_error_test.go +++ b/src/pkg/appkey/write_error_test.go @@ -338,3 +338,36 @@ func TestWriteKeyFileAtomicReplacesUnwritableFile(t *testing.T) { t.Fatal("rejected writes must leave the key untouched") } } + +func TestHeldFingerprintsDoesNotRereadUnchangedKeys(t *testing.T) { + r := newTestResolver(t) + if err := os.MkdirAll(r.DataDir, 0o700); err != nil { + t.Fatal(err) + } + path := filepath.Join(r.DataDir, "gh-app-key-42.pem") + writeTestAppKey(t, path) + + reads := 0 + orig := fingerprintFromFile + fingerprintFromFile = func(p string) (string, error) { + reads++ + return orig(p) + } + t.Cleanup(func() { fingerprintFromFile = orig }) + + first := r.HeldFingerprints() + second := r.HeldFingerprints() + if first["42"] == "" || second["42"] != first["42"] { + t.Fatalf("fingerprints = %v then %v", first, second) + } + if reads != 1 { + t.Fatalf("key file read %d times across two scans, want 1", reads) + } + + if err := os.Remove(path); err != nil { + t.Fatal(err) + } + if got := r.HeldFingerprints(); got != nil { + t.Fatalf("removed key still reported: %v", got) + } +} diff --git a/src/pkg/config/agent_config.go b/src/pkg/config/agent_config.go index d918e1ee2f..3c9daeda15 100644 --- a/src/pkg/config/agent_config.go +++ b/src/pkg/config/agent_config.go @@ -118,14 +118,22 @@ func SplitCadenceTargetKey(key string) (agent, repo string) { // legal config, and refusing to boot on it would turn a diagnostic into an // outage. func AgentSandboxGateWarnings(cfg *Config) []string { + return AgentSandboxGateReport(cfg).Warnings +} + +// AgentSandboxGateReport is AgentSandboxGateWarnings plus the preflight +// findings kept apart per runtime, so callers can attribute them to the agents +// on that runtime without parsing strings. Warnings is the same flat list. +func AgentSandboxGateReport(cfg *Config) SandboxGateReport { if cfg == nil || !cfg.AgentSandbox.Enabled { // The sandbox is off globally. That is the documented default and the // posture the docs describe; it is not a misconfiguration. - return nil + return SandboxGateReport{} } - var optedIn, noImage, noClaim, noGateway, badRuntime []string + var optedIn, noImage, noClaim, noGateway, noDigest, badRuntime []string var openshellBlocks []SandboxOpenShellConfig + var substrateBlocks []SandboxSubstrateConfig for name, a := range cfg.Agents { if !a.SandboxEnabled(cfg.AgentSandbox) { continue @@ -150,12 +158,21 @@ func AgentSandboxGateWarnings(cfg *Config) []string { if strings.TrimSpace(os.GatewayURL) != "" { openshellBlocks = append(openshellBlocks, os) } + case rt == SandboxRuntimeSubstrate: + sb := a.SandboxSubstrate(cfg.AgentSandbox) + // Substrate refuses tag-only images; the agent image is usually a + // tag, so the substrate block must carry the digest-pinned one. + if !strings.Contains(sb.Image, "@sha256:") { + noDigest = append(noDigest, name) + } + substrateBlocks = append(substrateBlocks, sb) } } sort.Strings(optedIn) sort.Strings(noImage) sort.Strings(noClaim) sort.Strings(noGateway) + sort.Strings(noDigest) sort.Strings(badRuntime) var out []string @@ -164,7 +181,7 @@ func AgentSandboxGateWarnings(cfg *Config) []string { "agent_sandbox.enabled is true but NO agent is opted in, so the sandbox is inert and all %d agent(s) still run unconfined on the tmux path — "+ "the per-agent gate is separate: set `sandbox: {enabled: true}` on each agent that should be sandboxed (#4918)", len(cfg.Agents))) - return out + return SandboxGateReport{Warnings: out} } if len(noImage) > 0 { out = append(out, fmt.Sprintf( @@ -190,6 +207,12 @@ func AgentSandboxGateWarnings(cfg *Config) []string { "set agent_sandbox.openshell.gateway_url (or the per-agent sandbox.openshell.gateway_url) to the OpenShell gateway's gRPC address (ADR-0026)", strings.Join(noGateway, ", "))) } + if len(noDigest) > 0 { + out = append(out, fmt.Sprintf( + "agent(s) %s use the substrate sandbox runtime but resolve no digest-pinned substrate.image; Substrate refuses tag-only images and every kick will fail — "+ + "set agent_sandbox.substrate.image (or the per-agent sandbox.substrate.image) to the sandbox image's …@sha256:… reference (ADR-0026 phase 2)", + strings.Join(noDigest, ", "))) + } if len(optedIn) < len(cfg.Agents) { out = append(out, fmt.Sprintf( "agent_sandbox.enabled is true but only %d of %d agent(s) are opted in (%s); the rest still run unconfined on the tmux path (#4918)", @@ -197,8 +220,11 @@ func AgentSandboxGateWarnings(cfg *Config) []string { } // Last, because it may reach the network (bounded and cached): the static // warnings above must not wait on a gateway. - out = append(out, cruxconfig.OpenShellPreflightWarnings(openshellBlocks)...) - return out + openshellWarnings := cruxconfig.OpenShellPreflightWarnings(openshellBlocks) + substrateWarnings := cruxconfig.SubstratePreflightWarnings(substrateBlocks) + out = append(out, openshellWarnings...) + out = append(out, substrateWarnings...) + return SandboxGateReport{Warnings: out, OpenShell: openshellWarnings, Substrate: substrateWarnings} } // CadenceIsOperatorOwned reports whether an operator explicitly set the diff --git a/src/pkg/config/agent_sandbox_report.go b/src/pkg/config/agent_sandbox_report.go new file mode 100644 index 0000000000..b21a42c1b0 --- /dev/null +++ b/src/pkg/config/agent_sandbox_report.go @@ -0,0 +1,45 @@ +package config + +import "sync" + +// SandboxGateReport is the result of AgentSandboxGateReport. OpenShell and +// Substrate are the preflight findings for agents on that runtime; both are +// also included in Warnings. +type SandboxGateReport struct { + Warnings []string + OpenShell []string + Substrate []string +} + +var ( + sandboxGateReportMu sync.RWMutex + sandboxGateReport SandboxGateReport +) + +// StoreSandboxGateReport records the report computed at boot/reload so +// readers such as the status builder never run a preflight themselves. +func StoreSandboxGateReport(r SandboxGateReport) { + sandboxGateReportMu.Lock() + defer sandboxGateReportMu.Unlock() + sandboxGateReport = r +} + +// LastSandboxGateReport returns the report last stored by StoreSandboxGateReport. +func LastSandboxGateReport() SandboxGateReport { + sandboxGateReportMu.RLock() + defer sandboxGateReportMu.RUnlock() + return sandboxGateReport +} + +// SandboxPreflightWarningsFor returns the stored preflight warnings that apply +// to an agent on the given sandbox runtime. +func SandboxPreflightWarningsFor(runtime string) []string { + r := LastSandboxGateReport() + switch runtime { + case SandboxRuntimeOpenShell: + return r.OpenShell + case SandboxRuntimeSubstrate: + return r.Substrate + } + return nil +} diff --git a/src/pkg/config/agent_sandbox_report_test.go b/src/pkg/config/agent_sandbox_report_test.go new file mode 100644 index 0000000000..f1e0115515 --- /dev/null +++ b/src/pkg/config/agent_sandbox_report_test.go @@ -0,0 +1,27 @@ +package config + +import ( + "reflect" + "testing" +) + +func TestSandboxPreflightWarningsFor(t *testing.T) { + t.Cleanup(func() { StoreSandboxGateReport(SandboxGateReport{}) }) + StoreSandboxGateReport(SandboxGateReport{ + OpenShell: []string{"o"}, + Substrate: []string{"s"}, + }) + for _, tc := range []struct { + runtime string + want []string + }{ + {SandboxRuntimeOpenShell, []string{"o"}}, + {SandboxRuntimeSubstrate, []string{"s"}}, + {SandboxRuntimeJob, nil}, + {"", nil}, + } { + if got := SandboxPreflightWarningsFor(tc.runtime); !reflect.DeepEqual(got, tc.want) { + t.Errorf("SandboxPreflightWarningsFor(%q) = %v, want %v", tc.runtime, got, tc.want) + } + } +} diff --git a/src/pkg/config/agent_sandbox_substrate_test.go b/src/pkg/config/agent_sandbox_substrate_test.go new file mode 100644 index 0000000000..3490aa1f61 --- /dev/null +++ b/src/pkg/config/agent_sandbox_substrate_test.go @@ -0,0 +1,115 @@ +package config + +import ( + "strings" + "testing" + + cruxconfig "github.com/hivecommons/hive/crux/config" + "gopkg.in/yaml.v3" +) + +// sandbox.runtime: substrate (ADR-0026 phase 2) — parse, merge semantics and +// the boot warnings that keep a misconfigured runtime from failing silently. + +func TestSandboxSubstrateParsesAndMerges(t *testing.T) { + src := ` +agent_sandbox: + enabled: true + image: img + runtime: substrate + substrate: + endpoint: api.ate-system.svc:443 + atespace: hive + token_env: ATE_TOKEN + image: ghcr.io/hivecommons/sandbox@sha256:0000000000000000000000000000000000000000000000000000000000000000 + allow_hosts: [litellm.example.com] + timeout_s: 600 + snapshot_on_idle: true +agents: + deps: + sandbox: + enabled: true + substrate: + allow_hosts: [hub.example.org] + worker_selector: { class: gvisor } +` + var cfg Config + if err := yaml.Unmarshal([]byte(src), &cfg); err != nil { + t.Fatal(err) + } + a := cfg.Agents["deps"] + if got := a.SandboxRuntime(cfg.AgentSandbox); got != SandboxRuntimeSubstrate { + t.Fatalf("runtime = %q", got) + } + eff := a.SandboxSubstrate(cfg.AgentSandbox) + if eff.Endpoint != "api.ate-system.svc:443" || eff.Atespace != "hive" || eff.TokenEnv != "ATE_TOKEN" || eff.TimeoutS != 600 || !eff.SnapshotOnIdle || !strings.Contains(eff.Image, "@sha256:") { + t.Errorf("cluster facts must fall back to the global block: %+v", eff) + } + if len(eff.AllowHosts) != 1 || eff.AllowHosts[0] != "hub.example.org" { + t.Errorf("per-agent allow_hosts must replace the global list: %v", eff.AllowHosts) + } + if eff.WorkerSelector["class"] != "gvisor" { + t.Errorf("worker_selector = %v", eff.WorkerSelector) + } + other := AgentConfig{} + if got := other.SandboxSubstrate(cfg.AgentSandbox); len(got.AllowHosts) != 1 || got.AllowHosts[0] != "litellm.example.com" { + t.Errorf("agent without a block gets the global one: %+v", got) + } +} + +func TestSandboxSubstrateGateWarnings(t *testing.T) { + yes := true + cfg := &Config{ + AgentRuntimeConfig: AgentRuntimeConfig{ + AgentSandbox: AgentSandboxConfig{Enabled: true, Image: "img"}, + Agents: map[string]AgentConfig{ + "deps": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimeSubstrate}}, + }, + }, + } + cruxconfig.SetSubstratePreflight(func([]SandboxSubstrateConfig) []string { return nil }) + defer cruxconfig.SetSubstratePreflight(nil) + got := AgentSandboxGateWarnings(cfg) + if len(got) != 1 || !strings.Contains(got[0], "digest-pinned") { + t.Errorf("warnings = %q, want exactly one about the digest-pinned image", got) + } + cfg.AgentSandbox.Substrate = &SandboxSubstrateConfig{Image: "ghcr.io/x/y@sha256:" + strings.Repeat("a", 64)} + if got := AgentSandboxGateWarnings(cfg); len(got) != 0 { + t.Errorf("fully configured substrate runtime still warns: %q", got) + } +} + +func TestAgentSandboxGateWarningsIncludeSubstratePreflight(t *testing.T) { + yes := true + cfg := &Config{ + AgentRuntimeConfig: AgentRuntimeConfig{ + AgentSandbox: AgentSandboxConfig{ + Enabled: true, + Image: "img", + Substrate: &SandboxSubstrateConfig{Image: "ghcr.io/x/y@sha256:" + strings.Repeat("a", 64)}, + }, + Agents: map[string]AgentConfig{ + "a": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimeSubstrate}}, + "b": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimeSubstrate}}, + }, + }, + } + var calls, gotBlocks int + cruxconfig.SetSubstratePreflight(func(b []SandboxSubstrateConfig) []string { + calls++ + gotBlocks = len(b) + return []string{"substrate control API api.ate-system.svc:443 is unreachable"} + }) + defer cruxconfig.SetSubstratePreflight(nil) + got := AgentSandboxGateWarnings(cfg) + if len(got) != 1 || !strings.Contains(got[0], "unreachable") || calls != 1 || gotBlocks != 2 { + t.Errorf("warnings = %q calls=%d blocks=%d, want the preflight warning from one call over both agents", got, calls, gotBlocks) + } + + cfg.Agents = map[string]AgentConfig{"p": {Sandbox: &AgentSandboxOverride{Enabled: &yes, Runtime: SandboxRuntimePodman}}} + calls = 0 + AgentSandboxGateWarnings(cfg) + if calls != 0 { + t.Errorf("preflight ran with no substrate agent") + } +} diff --git a/src/pkg/config/atomic_write.go b/src/pkg/config/atomic_write.go new file mode 100644 index 0000000000..bdd969964d --- /dev/null +++ b/src/pkg/config/atomic_write.go @@ -0,0 +1,116 @@ +package config + +import ( + "errors" + "fmt" + "os" + "path/filepath" +) + +// writeFileAtomic replaces the contents of path without ever exposing a +// truncated or partial file to a concurrent reader or to the next boot. +// +// Motivation (#17862): Save used O_WRONLY|O_TRUNC in place on the PVC runtime +// config. A SIGKILL (liveness probe) landing between the truncate and the +// write left a 0-byte /data/hive.yaml.runtime; the entrypoint's `-s` check +// then fell back to a months-stale /data/hive.yaml.bak and a hive booted +// with a three-month-old ACMM level and roster. +// +// Strategy: write data to a temp file in the TARGET's directory, fsync it, +// then rename it onto the target. Two constraints shape the details: +// +// - path may be a symlink (hosted spokes link /etc/hive/hive.yaml to the +// PVC runtime config, #17468). The rename must land on the link's +// TARGET, not replace the link with a regular file, so the final path is +// resolved with EvalSymlinks first. Pinned by TestSaveThroughSymlinkKeepsLink. +// - path may be a single-file Docker bind mount, where rename over the +// mount point fails (EBUSY) and would also swap the inode the host sees. +// When the rename fails, fall back to the previous in-place write so +// those deployments keep working exactly as before; they never had +// atomicity and lose nothing here. +// +// mode is applied to the temp file before the rename, so a pre-existing +// world-readable inode does not carry its bits through (WriteFile's mode +// only applies on create). The fallback path chmods explicitly for the same +// reason. +func writeFileAtomic(path string, data []byte, mode os.FileMode) error { + target := path + if resolved, err := filepath.EvalSymlinks(path); err == nil { + target = resolved + } else if !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("resolving %s: %w", path, err) + } + + dir := filepath.Dir(target) + tmp, err := os.CreateTemp(dir, "."+filepath.Base(target)+".tmp-*") + if err != nil { + return fmt.Errorf("creating temp file in %s: %w", dir, err) + } + tmpPath := tmp.Name() + cleanup := func() { _ = os.Remove(tmpPath) } + + if err := tmp.Chmod(mode); err != nil { + _ = tmp.Close() + cleanup() + return fmt.Errorf("chmod temp file %s: %w", tmpPath, err) + } + if _, err := tmp.Write(data); err != nil { + _ = tmp.Close() + cleanup() + return fmt.Errorf("writing temp file %s: %w", tmpPath, err) + } + if err := tmp.Sync(); err != nil { + _ = tmp.Close() + cleanup() + return fmt.Errorf("syncing temp file %s: %w", tmpPath, err) + } + if err := tmp.Close(); err != nil { + cleanup() + return fmt.Errorf("closing temp file %s: %w", tmpPath, err) + } + + if err := os.Rename(tmpPath, target); err != nil { + cleanup() + // Bind-mounted single file (or any filesystem that refuses the + // rename): keep the inode and write in place, as Save always did. + if inPlaceErr := writeFileInPlace(target, data, mode); inPlaceErr != nil { + return fmt.Errorf("renaming %s onto %s: %w (in-place fallback also failed: %v)", tmpPath, target, err, inPlaceErr) + } + return nil + } + syncDir(dir) + return nil +} + +// writeFileInPlace is the pre-#17862 O_TRUNC write, kept only as the fallback +// for targets that cannot be renamed over. It is NOT crash-safe. +func writeFileInPlace(target string, data []byte, mode os.FileMode) error { + f, err := os.OpenFile(target, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, mode) + if err != nil { + return err + } + if _, err := f.Write(data); err != nil { + _ = f.Close() + return err + } + if err := f.Sync(); err != nil { + _ = f.Close() + return err + } + if err := f.Close(); err != nil { + return err + } + return os.Chmod(target, mode) +} + +// syncDir fsyncs a directory so the rename that just happened in it is +// durable. Best-effort: some filesystems (and all of Windows) refuse to open +// a directory for sync, and a failure here cannot un-do the rename. +func syncDir(dir string) { + d, err := os.Open(dir) + if err != nil { + return + } + _ = d.Sync() + _ = d.Close() +} diff --git a/src/pkg/config/atomic_write_test.go b/src/pkg/config/atomic_write_test.go new file mode 100644 index 0000000000..dfdb561e23 --- /dev/null +++ b/src/pkg/config/atomic_write_test.go @@ -0,0 +1,288 @@ +package config + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +// TestWriteFileAtomicReplacesContentAndMode covers the plain case: an +// existing world-readable file is replaced wholesale and ends up with the +// requested mode, and no temp file is left behind. +func TestWriteFileAtomicReplacesContentAndMode(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "hive.yaml.runtime") + if err := os.WriteFile(path, []byte("old: content\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeFileAtomic(path, []byte("new: content\n"), 0o600); err != nil { + t.Fatalf("writeFileAtomic: %v", err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != "new: content\n" { + t.Fatalf("content = %q, want new content", got) + } + fi, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if fi.Mode().Perm() != 0o600 { + t.Fatalf("mode = %o, want 0600 (pre-existing 0644 bits must not survive)", fi.Mode().Perm()) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + for _, e := range entries { + if strings.Contains(e.Name(), ".tmp-") { + t.Fatalf("temp file %s left behind", e.Name()) + } + } +} + +// TestWriteFileAtomicCreatesMissingFile pins that a missing target is +// created (Save's old create fallback) rather than failing on EvalSymlinks. +func TestWriteFileAtomicCreatesMissingFile(t *testing.T) { + path := filepath.Join(t.TempDir(), "hive.yaml") + if err := writeFileAtomic(path, []byte("a: 1\n"), 0o644); err != nil { + t.Fatalf("writeFileAtomic on missing file: %v", err) + } + if got, _ := os.ReadFile(path); string(got) != "a: 1\n" { + t.Fatalf("content = %q", got) + } +} + +// TestWriteFileAtomicThroughSymlinkKeepsLink is the #17468 constraint at the +// writer level: the rename must land on the link TARGET so the symlink the +// copy-config init container created survives, and the target carries the +// new bytes. +func TestWriteFileAtomicThroughSymlinkKeepsLink(t *testing.T) { + dir := t.TempDir() + target := filepath.Join(dir, "data", "hive.yaml.runtime") + link := filepath.Join(dir, "etc", "hive.yaml") + for _, d := range []string{filepath.Dir(target), filepath.Dir(link)} { + if err := os.MkdirAll(d, 0o755); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(target, []byte("old\n"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, link); err != nil { + t.Fatal(err) + } + if err := writeFileAtomic(link, []byte("new\n"), 0o600); err != nil { + t.Fatalf("writeFileAtomic through link: %v", err) + } + fi, err := os.Lstat(link) + if err != nil { + t.Fatal(err) + } + if fi.Mode()&os.ModeSymlink == 0 { + t.Fatalf("%s is no longer a symlink — rename replaced the link instead of its target", link) + } + if got, _ := os.ReadFile(target); string(got) != "new\n" { + t.Fatalf("target content = %q, want new", got) + } + // Nothing may be left in the link's directory: the temp file must be + // created next to the TARGET, or the rename would cross directories. + entries, _ := os.ReadDir(filepath.Dir(link)) + if len(entries) != 1 { + t.Fatalf("link dir has %d entries, want just the link", len(entries)) + } +} + +// TestSaveSurvivesKillMidWrite is the #17862 regression. It cannot SIGKILL +// the test process, so it pins the property that makes a kill harmless: +// at no point during Save does the runtime config path refer to a file +// with fewer bytes than the previous complete config. A temp-file+rename +// writer satisfies this trivially; the old O_TRUNC in-place write fails it +// because the file is observably 0 bytes between truncate and write. +// +// The check is made by racing a reader against repeated Saves and +// asserting it never observes an empty or partial file. +func TestSaveSurvivesKillMidWrite(t *testing.T) { + dir := t.TempDir() + cfgPath := filepath.Join(dir, "hive.yaml") + runtimePath := filepath.Join(dir, "hive.yaml.runtime") + origOverlay, origRuntime := DashboardOverlayFile, RuntimeConfigFile + DashboardOverlayFile = filepath.Join(dir, "hive.yaml.dashboard") + RuntimeConfigFile = runtimePath + t.Cleanup(func() { DashboardOverlayFile, RuntimeConfigFile = origOverlay, origRuntime }) + + level := 6 + cfg := &Config{ + SourcePath: cfgPath, + Project: ProjectConfig{Org: "testorg", Name: "test"}, + AgentRuntimeConfig: AgentRuntimeConfig{ + Agents: map[string]AgentConfig{"scanner": {Role: "scanner"}}, + }, + ACMMLevel: &level, + } + if err := cfg.Save(); err != nil { + t.Fatalf("initial Save: %v", err) + } + full, err := os.ReadFile(runtimePath) + if err != nil || len(full) == 0 { + t.Fatalf("initial runtime config unreadable: %v", err) + } + + stop := make(chan struct{}) + bad := make(chan string, 1) + go func() { + for { + select { + case <-stop: + return + default: + } + for _, p := range []string{runtimePath, cfgPath} { + got, err := os.ReadFile(p) + if err != nil { + continue // a rename window may briefly ENOENT on some filesystems; never partial + } + if len(got) < len(full) { + select { + case bad <- p: + default: + } + return + } + } + } + }() + for i := 0; i < 200; i++ { + if err := cfg.Save(); err != nil { + t.Fatalf("Save %d: %v", i, err) + } + } + close(stop) + select { + case p := <-bad: + t.Fatalf("reader observed a truncated/partial config at %s during Save — write is not atomic", p) + default: + } +} + +func TestWriteFileAtomicMissingDirFails(t *testing.T) { + path := filepath.Join(t.TempDir(), "no-such-dir", "hive.yaml") + err := writeFileAtomic(path, []byte("x"), 0o600) + if err == nil { + t.Fatal("expected error when target directory does not exist") + } + if !strings.Contains(err.Error(), "creating temp file") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestWriteFileAtomicResolveErrorIsReported(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "regular") + if err := os.WriteFile(file, []byte("f"), 0o600); err != nil { + t.Fatal(err) + } + // Using a regular file as a path component makes EvalSymlinks fail with + // ENOTDIR, which is not ErrNotExist and must surface as a resolve error. + err := writeFileAtomic(filepath.Join(file, "hive.yaml"), []byte("x"), 0o600) + if err == nil { + t.Fatal("expected resolve error") + } + if !strings.Contains(err.Error(), "resolving") { + t.Fatalf("unexpected error: %v", err) + } +} + +func TestWriteFileAtomicDanglingSymlinkIsReplaced(t *testing.T) { + dir := t.TempDir() + link := filepath.Join(dir, "hive.yaml") + if err := os.Symlink(filepath.Join(dir, "missing-target"), link); err != nil { + t.Skipf("symlink not supported: %v", err) + } + if err := writeFileAtomic(link, []byte("fresh"), 0o600); err != nil { + t.Fatalf("writeFileAtomic: %v", err) + } + got, err := os.ReadFile(link) + if err != nil { + t.Fatal(err) + } + if string(got) != "fresh" { + t.Fatalf("got %q", got) + } +} + +func TestWriteFileAtomicRenameFailureFallsBackAndReportsBoth(t *testing.T) { + dir := t.TempDir() + // A non-empty directory as the target: rename of a file onto it fails, + // and the in-place O_TRUNC fallback cannot open a directory for writing. + target := filepath.Join(dir, "hive.yaml") + if err := os.MkdirAll(filepath.Join(target, "child"), 0o700); err != nil { + t.Fatal(err) + } + err := writeFileAtomic(target, []byte("x"), 0o600) + if err == nil { + t.Fatal("expected rename + fallback failure") + } + if !strings.Contains(err.Error(), "in-place fallback also failed") { + t.Fatalf("unexpected error: %v", err) + } + entries, _ := os.ReadDir(dir) + for _, e := range entries { + if strings.HasPrefix(e.Name(), ".hive.yaml.tmp-") { + t.Fatalf("temp file %s leaked after failure", e.Name()) + } + } +} + +func TestWriteFileInPlaceWritesAndSetsMode(t *testing.T) { + path := filepath.Join(t.TempDir(), "hive.yaml") + if err := os.WriteFile(path, []byte("old-longer-content"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeFileInPlace(path, []byte("new"), 0o600); err != nil { + t.Fatalf("writeFileInPlace: %v", err) + } + got, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if string(got) != "new" { + t.Fatalf("got %q, want truncated rewrite", got) + } + st, err := os.Stat(path) + if err != nil { + t.Fatal(err) + } + if st.Mode().Perm() != 0o600 { + t.Fatalf("mode = %o, want 600", st.Mode().Perm()) + } +} + +func TestWriteFileInPlaceOpenFailure(t *testing.T) { + path := filepath.Join(t.TempDir(), "missing", "hive.yaml") + if err := writeFileInPlace(path, []byte("x"), 0o600); err == nil { + t.Fatal("expected open failure for missing directory") + } +} + +func TestSyncDirToleratesMissingDir(t *testing.T) { + syncDir(filepath.Join(t.TempDir(), "nope")) + syncDir(t.TempDir()) +} + +func TestSourceConfigModePreservesExistingOrDefaults(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "hive.yaml") + if err := os.WriteFile(path, []byte("a: 1\n"), 0o600); err != nil { + t.Fatal(err) + } + if got := sourceConfigMode(path); got != 0o600 { + t.Fatalf("existing mode = %o, want 600", got) + } + if got := sourceConfigMode(filepath.Join(dir, "absent.yaml")); got != 0o644 { + t.Fatalf("default mode = %o, want 644", got) + } +} diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index 262b045911..40d213ba84 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -74,7 +74,10 @@ type Config struct { Data DataConfig `yaml:"data"` Deployment DeploymentConfig `yaml:"deployment,omitempty" json:"deployment,omitempty"` Knowledge KnowledgeConfig `yaml:"knowledge"` - Hub HubConfig `yaml:"hub"` + // Inbox routes voice-recorder items (Plaud, Wispr Flow) to spek RFCs or + // dibs from the dashboard Inbox. + Inbox InboxConfig `yaml:"inbox,omitempty" json:"inbox,omitempty"` + Hub HubConfig `yaml:"hub"` // Issues tunes issue lifecycle automation. Zero value keeps the safe // defaults: close issues after their fix PR merges, and backfill hourly. Issues IssuesConfig `yaml:"issues,omitempty" json:"issues,omitempty"` diff --git a/src/pkg/config/crux_aliases.go b/src/pkg/config/crux_aliases.go index 3898aaa7b2..72838f2c4e 100644 --- a/src/pkg/config/crux_aliases.go +++ b/src/pkg/config/crux_aliases.go @@ -39,6 +39,7 @@ type ( SandboxJobToleration = cruxconfig.SandboxJobToleration SandboxJobVolume = cruxconfig.SandboxJobVolume SandboxOpenShellConfig = cruxconfig.SandboxOpenShellConfig + SandboxSubstrateConfig = cruxconfig.SandboxSubstrateConfig StatsDisplayEntry = cruxconfig.StatsDisplayEntry ToolApprovalConfig = cruxconfig.ToolApprovalConfig ToolApprovalRule = cruxconfig.ToolApprovalRule @@ -147,6 +148,7 @@ const ( RotationHeadroomSourceCCLeft = cruxconfig.RotationHeadroomSourceCCLeft SandboxRuntimeJob = cruxconfig.SandboxRuntimeJob SandboxRuntimeOpenShell = cruxconfig.SandboxRuntimeOpenShell + SandboxRuntimeSubstrate = cruxconfig.SandboxRuntimeSubstrate SandboxRuntimePodman = cruxconfig.SandboxRuntimePodman SelfMergeMinACMMLevel = cruxconfig.SelfMergeMinACMMLevel WritableBobAPIKeyFile = cruxconfig.WritableBobAPIKeyFile diff --git a/src/pkg/config/inbox_config.go b/src/pkg/config/inbox_config.go new file mode 100644 index 0000000000..a40f0b312e --- /dev/null +++ b/src/pkg/config/inbox_config.go @@ -0,0 +1,122 @@ +package config + +import ( + "fmt" + "regexp" + "sort" + "strings" +) + +// Inbox destinations. `knowledge` is implicit: every item is already a +// Knowledge fact when it reaches the Inbox. +const ( + InboxDestinationSpekRFC = "spek_rfc" + InboxDestinationDibs = "dibs" + InboxDestinationDismiss = "dismiss" +) + +// DefaultInboxSources are the connector types whose facts surface as Inbox +// items when inbox.sources is empty. +var DefaultInboxSources = []string{"plaud", "wisprflow"} + +// InboxConfig is the `inbox:` block: which connector types feed the Inbox, +// how items are auto-routed, and how each destination is reached. +type InboxConfig struct { + // Sources lists connector types (plaud, wisprflow, ...) whose facts are + // Inbox items. Empty uses DefaultInboxSources. + Sources []string `yaml:"sources,omitempty" json:"sources,omitempty"` + // Rules are evaluated in order against every new item; the first match + // routes it automatically. + Rules []InboxRule `yaml:"rules,omitempty" json:"rules,omitempty"` + Spek InboxSpek `yaml:"spek,omitempty" json:"spek,omitempty"` + Dibs InboxDibs `yaml:"dibs,omitempty" json:"dibs,omitempty"` +} + +// InboxRule routes items whose source and title/tag match. +type InboxRule struct { + // Source restricts the rule to one connector type; empty matches all. + Source string `yaml:"source,omitempty" json:"source,omitempty"` + // TitlePrefix matches case-insensitively against the item title. + TitlePrefix string `yaml:"title_prefix,omitempty" json:"title_prefix,omitempty"` + // Tag matches any item tag exactly (case-insensitive). + Tag string `yaml:"tag,omitempty" json:"tag,omitempty"` + // Destination is spek_rfc, dibs or dismiss. + Destination string `yaml:"destination" json:"destination"` + // Repo (owner/repo) overrides inbox.spek.repo for spek_rfc. + Repo string `yaml:"repo,omitempty" json:"repo,omitempty"` +} + +// InboxSpek configures the spek_rfc destination. +type InboxSpek struct { + // Repo is the default owner/repo RFCs are filed to. + Repo string `yaml:"repo,omitempty" json:"repo,omitempty"` + // Binary is the spektacular executable (default runs.spektacular.binary). + Binary string `yaml:"binary,omitempty" json:"binary,omitempty"` + // Agent names a hive agent to kick with a spek-new/publish prompt when + // the binary is unavailable in the hive image. Empty picks the first + // enabled, unpaused agent by name (see InboxSpekAgent). + Agent string `yaml:"agent,omitempty" json:"agent,omitempty"` +} + +// InboxSpekAgent returns the agent the spek_rfc destination falls back to +// when the spektacular binary is absent: inbox.spek.agent if set, else the +// alphabetically first enabled, unpaused agent, else "". +func (c *Config) InboxSpekAgent() string { + if a := strings.TrimSpace(c.Inbox.Spek.Agent); a != "" { + return a + } + names := make([]string, 0, len(c.Agents)) + for name, a := range c.Agents { + if a.Enabled && !a.Paused { + names = append(names, name) + } + } + sort.Strings(names) + if len(names) == 0 { + return "" + } + return names[0] +} + +// InboxDibs configures the dibs destination. +type InboxDibs struct { + // URL is the dibs instance base URL. + URL string `yaml:"url,omitempty" json:"url,omitempty"` + // TokenEnv names the environment variable holding the hub bearer token. + TokenEnv string `yaml:"token_env,omitempty" json:"token_env,omitempty"` +} + +var inboxRepoRe = regexp.MustCompile(`^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$`) + +// EffectiveSources returns Sources or DefaultInboxSources. +func (c InboxConfig) EffectiveSources() []string { + if len(c.Sources) == 0 { + return append([]string(nil), DefaultInboxSources...) + } + return c.Sources +} + +// ValidateInbox rejects unknown destinations, malformed repos and rules +// without a filter (a filter-less rule would route every item). +func ValidateInbox(c InboxConfig) error { + if c.Spek.Repo != "" && !inboxRepoRe.MatchString(c.Spek.Repo) { + return fmt.Errorf("inbox.spek.repo %q: must be owner/repo", c.Spek.Repo) + } + if c.Dibs.URL != "" && !strings.HasPrefix(c.Dibs.URL, "http://") && !strings.HasPrefix(c.Dibs.URL, "https://") { + return fmt.Errorf("inbox.dibs.url %q: must start with http:// or https://", c.Dibs.URL) + } + for i, r := range c.Rules { + switch r.Destination { + case InboxDestinationSpekRFC, InboxDestinationDibs, InboxDestinationDismiss: + default: + return fmt.Errorf("inbox.rules[%d].destination %q: must be %s, %s or %s", i, r.Destination, InboxDestinationSpekRFC, InboxDestinationDibs, InboxDestinationDismiss) + } + if strings.TrimSpace(r.TitlePrefix) == "" && strings.TrimSpace(r.Tag) == "" { + return fmt.Errorf("inbox.rules[%d]: title_prefix or tag is required", i) + } + if r.Repo != "" && !inboxRepoRe.MatchString(r.Repo) { + return fmt.Errorf("inbox.rules[%d].repo %q: must be owner/repo", i, r.Repo) + } + } + return nil +} diff --git a/src/pkg/config/inbox_config_default_agent_test.go b/src/pkg/config/inbox_config_default_agent_test.go new file mode 100644 index 0000000000..a3193c4e13 --- /dev/null +++ b/src/pkg/config/inbox_config_default_agent_test.go @@ -0,0 +1,51 @@ +package config + +import "testing" + +func TestInboxSpekAgentDefaultsToFirstEnabledAgent(t *testing.T) { + c := &Config{AgentRuntimeConfig: AgentRuntimeConfig{Agents: map[string]AgentConfig{ + "zed": {Enabled: true}, + "alpha": {Enabled: true, Paused: true}, + "bravo": {Enabled: true}, + "charly": {Enabled: false}, + }}} + if got := c.InboxSpekAgent(); got != "bravo" { + t.Fatalf("default agent = %q, want bravo (first enabled, unpaused)", got) + } + c.Inbox.Spek.Agent = " custom " + if got := c.InboxSpekAgent(); got != "custom" { + t.Fatalf("explicit agent = %q, want custom", got) + } + if got := (&Config{}).InboxSpekAgent(); got != "" { + t.Fatalf("no agents should give empty, got %q", got) + } +} + +func TestInboxEffectiveSourcesAndValidate(t *testing.T) { + if got := (InboxConfig{}).EffectiveSources(); len(got) != len(DefaultInboxSources) || got[0] != DefaultInboxSources[0] { + t.Fatalf("default sources = %v", got) + } + if got := (InboxConfig{Sources: []string{"wisprflow"}}).EffectiveSources(); len(got) != 1 || got[0] != "wisprflow" { + t.Fatalf("explicit sources = %v", got) + } + ok := InboxConfig{ + Spek: InboxSpek{Repo: "o/r"}, + Dibs: InboxDibs{URL: "https://dibs.example"}, + Rules: []InboxRule{{TitlePrefix: "RFC:", Destination: InboxDestinationSpekRFC, Repo: "o/r"}, {Tag: "idea", Destination: InboxDestinationDibs}}, + } + if err := ValidateInbox(ok); err != nil { + t.Fatalf("valid config rejected: %v", err) + } + bad := map[string]InboxConfig{ + "spek repo": {Spek: InboxSpek{Repo: "not-a-repo"}}, + "dibs url": {Dibs: InboxDibs{URL: "dibs.example"}}, + "rule destination": {Rules: []InboxRule{{TitlePrefix: "x", Destination: "email"}}}, + "rule filter": {Rules: []InboxRule{{Destination: InboxDestinationDismiss}}}, + "rule repo": {Rules: []InboxRule{{TitlePrefix: "x", Destination: InboxDestinationSpekRFC, Repo: "nope"}}}, + } + for name, c := range bad { + if err := ValidateInbox(c); err == nil { + t.Errorf("%s: expected an error", name) + } + } +} diff --git a/src/pkg/config/review_config.go b/src/pkg/config/review_config.go index a273596989..83bda632f7 100644 --- a/src/pkg/config/review_config.go +++ b/src/pkg/config/review_config.go @@ -167,6 +167,20 @@ type ReviewConfig struct { // rather than left to infer them from the tree; a hive can only get that // by writing it down. An entry that is blank means "use the built-in". PerspectivePrompts map[string]string `yaml:"perspective_prompts,omitempty" json:"perspective_prompts,omitempty"` + // PerspectivesSource live-links the perspective names and focus text to a + // kind: ReviewPerspectives file in a GitHub repo (hivecommons/hive#17975), + // so maintainers can change the set through ordinary PRs. When it is set + // and resolved it wins over Perspectives/PerspectivePrompts, which remain + // the fallback. A partially filled source (no owner, repo or path) is + // treated as unset, matching DefinitionSourceConfig.IsSet. Only names and + // focus are sourced: combined_perspectives, plan_match and + // max_perspectives_per_pr stay operator-owned. + PerspectivesSource *DefinitionSourceConfig `yaml:"perspectives_source,omitempty" json:"perspectives_source,omitempty"` + // LinkedPerspectives is the set most recently resolved from + // PerspectivesSource. It is runtime-only and never persisted, so a fetched + // document can never be written back into the overlay. Read the effective + // set through EffectivePerspectives, never this field directly. + LinkedPerspectives *LinkedPerspectives `yaml:"-" json:"-"` // PerspectiveModels pins the backend/model one perspective's swarm review // runs with, keyed by perspective name. An entry replaces the reviewer // agent's review_models pool for that perspective only; the agent's @@ -398,6 +412,30 @@ type ReviewConfig struct { Reviewers ReviewersConfig `yaml:"reviewers,omitempty" json:"reviewers,omitzero"` } +// LinkedPerspectives is a review perspective set resolved from +// ReviewConfig.PerspectivesSource. It is already validated by the resolver; +// pkg/config deliberately does not import pkg/review to re-check it. +type LinkedPerspectives struct { + Names []string + Focus map[string]string + // Source is the provenance shown to operators, e.g. owner/repo/path@ref. + Source string + // Warning carries the last resolve failure while an older, still-valid set + // stays in effect. Empty when the latest document was applied. + Warning string +} + +// EffectivePerspectives returns the perspective names and focus text every +// consumer should use: the linked set when PerspectivesSource is set and has +// resolved, otherwise the inline Perspectives and PerspectivePrompts. With no +// source configured the result is exactly the inline values, unchanged. +func (r ReviewConfig) EffectivePerspectives() (names []string, focus map[string]string) { + if r.PerspectivesSource.IsSet() && r.LinkedPerspectives != nil { + return r.LinkedPerspectives.Names, r.LinkedPerspectives.Focus + } + return r.Perspectives, r.PerspectivePrompts +} + // ContributorPRsConfig is the owner-only gate for contributor PR maintenance. type ContributorPRsConfig struct { // BaseSync enables automatic update-branch attempts for fork PRs that are diff --git a/src/pkg/config/review_perspectives_source_test.go b/src/pkg/config/review_perspectives_source_test.go new file mode 100644 index 0000000000..85a7c66211 --- /dev/null +++ b/src/pkg/config/review_perspectives_source_test.go @@ -0,0 +1,109 @@ +package config + +import ( + "encoding/json" + "reflect" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestEffectivePerspectivesPrecedence(t *testing.T) { + source := &DefinitionSourceConfig{Owner: "acme", Repo: "widgets", Path: ".hive/review-perspectives.yaml"} + linked := &LinkedPerspectives{ + Names: []string{"correctness", "spyre-numerics"}, + Focus: map[string]string{"spyre-numerics": "fp16 accumulation order"}, + Source: "acme/widgets/.hive/review-perspectives.yaml", + } + inlineNames := []string{"security"} + inlineFocus := map[string]string{"security": "secrets in logs"} + + tests := []struct { + name string + cfg ReviewConfig + wantNames []string + wantFocus map[string]string + }{ + { + name: "source set and resolved uses linked set", + cfg: ReviewConfig{Perspectives: inlineNames, PerspectivePrompts: inlineFocus, PerspectivesSource: source, LinkedPerspectives: linked}, + wantNames: linked.Names, + wantFocus: linked.Focus, + }, + { + name: "source set but unresolved falls back to inline", + cfg: ReviewConfig{Perspectives: inlineNames, PerspectivePrompts: inlineFocus, PerspectivesSource: source}, + wantNames: inlineNames, + wantFocus: inlineFocus, + }, + { + name: "partial source is unset", + cfg: ReviewConfig{Perspectives: inlineNames, PerspectivePrompts: inlineFocus, PerspectivesSource: &DefinitionSourceConfig{Owner: "acme", Repo: "widgets"}, LinkedPerspectives: linked}, + wantNames: inlineNames, + wantFocus: inlineFocus, + }, + { + name: "source unset uses inline", + cfg: ReviewConfig{Perspectives: inlineNames, PerspectivePrompts: inlineFocus}, + wantNames: inlineNames, + wantFocus: inlineFocus, + }, + { + name: "nothing configured", + cfg: ReviewConfig{}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + names, focus := tc.cfg.EffectivePerspectives() + if !reflect.DeepEqual(names, tc.wantNames) { + t.Errorf("names = %v, want %v", names, tc.wantNames) + } + if !reflect.DeepEqual(focus, tc.wantFocus) { + t.Errorf("focus = %v, want %v", focus, tc.wantFocus) + } + }) + } +} + +func TestLinkedPerspectivesNotPersisted(t *testing.T) { + cfg := ReviewConfig{ + PerspectivesSource: &DefinitionSourceConfig{Owner: "acme", Repo: "widgets", Path: ".hive/review-perspectives.yaml"}, + LinkedPerspectives: &LinkedPerspectives{ + Names: []string{"linkedonlyname"}, + Focus: map[string]string{"linkedonlyname": "linkedonlyfocus"}, + Source: "linkedonlyprovenance", + Warning: "linkedonlywarning", + }, + } + y, err := yaml.Marshal(cfg) + if err != nil { + t.Fatalf("yaml.Marshal: %v", err) + } + j, err := json.Marshal(cfg) + if err != nil { + t.Fatalf("json.Marshal: %v", err) + } + for format, out := range map[string]string{"yaml": string(y), "json": string(j)} { + if !strings.Contains(out, "perspectives_source") { + t.Errorf("%s output lost perspectives_source:\n%s", format, out) + } + for _, leaked := range []string{"linked", "Linked"} { + if strings.Contains(out, leaked) { + t.Errorf("%s output persists linked data (%q):\n%s", format, leaked, out) + } + } + } + + var round ReviewConfig + if err := yaml.Unmarshal(y, &round); err != nil { + t.Fatalf("yaml.Unmarshal: %v", err) + } + if round.LinkedPerspectives != nil { + t.Errorf("LinkedPerspectives survived a yaml round-trip: %+v", round.LinkedPerspectives) + } + if !round.PerspectivesSource.IsSet() { + t.Errorf("PerspectivesSource lost in yaml round-trip: %+v", round.PerspectivesSource) + } +} diff --git a/src/pkg/config/save.go b/src/pkg/config/save.go index f70dbe8307..14fc4fe37d 100644 --- a/src/pkg/config/save.go +++ b/src/pkg/config/save.go @@ -135,18 +135,14 @@ func (c *Config) saveLocked() error { // layer this Save is about to persist. overlayErr := c.saveDashboardOverlay() - // Open the existing file (preserving its inode) rather than creating a - // temp file and renaming. Rename breaks Docker bind mounts because it - // replaces the inode — the host file is never updated, so acmm_level - // and other runtime changes are lost on container restart. + // Replace the primary config atomically (temp file + fsync + rename onto + // the resolved target) — see writeFileAtomic for why that is safe on the + // #17468 symlink and still works on single-file Docker bind mounts. // - // #17468: the same in-place write is what keeps hosted spokes durable. - // There the copy-config init container makes SourcePath - // (/etc/hive/hive.yaml, an emptyDir) a symlink to RuntimeConfigFile on - // the PVC; O_WRONLY|O_TRUNC follows the link and writes the PVC file, - // whereas a temp-file+rename would replace the link with a regular - // emptyDir file and silently revert the fix. Pinned by - // TestSaveThroughSymlinkKeepsLink — do not switch this to a rename. + // #17862: the previous O_WRONLY|O_TRUNC in-place write could be killed + // between truncate and write (liveness probe on a crash-looping hive), + // leaving a 0-byte runtime config that the entrypoint skipped in favour + // of a months-stale hive.yaml.bak. Pinned by TestSaveSurvivesKillMidWrite. // // #3961: a source-path failure must NOT abort the save. On deployments // that mount the config read-only (a ConfigMap mounted straight at @@ -160,23 +156,8 @@ func (c *Config) saveLocked() error { // writing the runtime layer, and report success iff the state will actually // survive a restart. var srcErr error - f, err := os.OpenFile(c.SourcePath, os.O_WRONLY|os.O_TRUNC, 0o644) - if err != nil { - // File may not exist yet — fall back to create. Continue below so - // the PVC backup and dashboard overlay are still written. - if writeErr := os.WriteFile(c.SourcePath, data, 0o644); writeErr != nil { - srcErr = fmt.Errorf("writing config (create fallback): %w", writeErr) - } - } else { - if _, err := f.Write(data); err != nil { - _ = f.Close() // best-effort cleanup; the write error is what's recorded - srcErr = fmt.Errorf("writing config: %w", err) - } else if err := f.Sync(); err != nil { - _ = f.Close() // best-effort cleanup; the sync error is what's recorded - srcErr = fmt.Errorf("syncing config: %w", err) - } else if err := f.Close(); err != nil { - srcErr = fmt.Errorf("closing config: %w", err) - } + if err := writeFileAtomic(c.SourcePath, data, sourceConfigMode(c.SourcePath)); err != nil { + srcErr = fmt.Errorf("writing config: %w", err) } // Persist the runtime config to the PVC. In K8s this is the boot input @@ -199,11 +180,11 @@ func (c *Config) saveLocked() error { // owner credential to every unprivileged agent user (#5331). if guardLivePVCPathUnderTest(runtimePath, defaultRuntimeConfigFile, "RuntimeConfigFile") { runtimeErr = errLivePVCPathGuarded - } else if err := os.WriteFile(runtimePath, data, 0o600); err != nil { + } else if err := writeFileAtomic(runtimePath, data, 0o600); err != nil { // Common cause: init container created the file as root, runtime user // can't overwrite. Remove and retry so runtime state is not silently lost. - _ = os.Remove(runtimePath) // best-effort; the retry's own WriteFile error is what's recorded below - if retryErr := os.WriteFile(runtimePath, data, 0o600); retryErr != nil { + _ = os.Remove(runtimePath) // best-effort; the retry's own error is what's recorded below + if retryErr := writeFileAtomic(runtimePath, data, 0o600); retryErr != nil { runtimeErr = retryErr log.Printf("[config] warning: failed to write PVC runtime config to %s (even after remove): %v", runtimePath, retryErr) } else { @@ -211,12 +192,6 @@ func (c *Config) saveLocked() error { } } else { log.Printf("[config] PVC runtime config written to %s", runtimePath) - // os.WriteFile's mode only applies when it CREATES the file; a - // pre-existing world-readable inode (every hive deployed before - // this fix) keeps its old 0644 bits, so tighten explicitly. - if chmodErr := os.Chmod(runtimePath, 0o600); chmodErr != nil { - log.Printf("[config] warning: failed to tighten permissions on %s: %v", runtimePath, chmodErr) - } } if srcErr == nil { @@ -235,6 +210,16 @@ func (c *Config) saveLocked() error { return srcErr } +// sourceConfigMode returns the mode Save should give the primary config +// path: the existing file's permission bits when it exists (an operator may +// have tightened them), else the historical 0644 default. +func sourceConfigMode(path string) os.FileMode { + if fi, err := os.Stat(path); err == nil { + return fi.Mode().Perm() + } + return 0o644 +} + // RuntimeConfigFile is where Save() persists the full runtime config on the // PVC. Its role differs by environment, which is exactly why the old // hive.yaml.bak name was misleading enough to cost debugging time: @@ -259,11 +244,12 @@ const defaultRuntimeConfigFile = "/data/hive.yaml.runtime" // RuntimeConfigFileLegacy is the pre-rename name of RuntimeConfigFile. // -// It is READ as a fallback and never written, renamed or removed: ~51 live -// hives carry only this file, and on Docker/LXC it is the single copy of -// their live configuration. Mutating it at boot could lose owner -// customisations with no warning, so the migration is copy-forward only — -// readers prefer RuntimeConfigFile and fall back to this one. +// It is READ as a fallback and never written by Save: ~51 live hives carry +// only this file, and on Docker/LXC it is the single copy of their live +// configuration. Readers prefer RuntimeConfigFile and fall back to this one. +// The entrypoint refreshes it from a good RuntimeConfigFile on every clean +// boot (#17862) — a fallback frozen at the Aug 2026 rename booted a +// months-old level/roster when a kill mid-save truncated the runtime file. // // Removable one release after every live hive has written the new name. const RuntimeConfigFileLegacy = "/data/hive.yaml.bak" diff --git a/src/pkg/config/testdata/agent_runtime_golden.json b/src/pkg/config/testdata/agent_runtime_golden.json index 990383aff5..85fe81d606 100644 --- a/src/pkg/config/testdata/agent_runtime_golden.json +++ b/src/pkg/config/testdata/agent_runtime_golden.json @@ -581,6 +581,10 @@ "OnError": "" } }, + "inbox": { + "spek": {}, + "dibs": {} + }, "Hub": { "Enabled": false, "URL": "https://hive.hivecommons.dev", diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 9904c4da73..d09e825597 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -274,6 +274,9 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { for _, w := range advisorWarnings { log.Printf("WARNING: %s", w) } + if err := ValidateInbox(c.Inbox); err != nil { + return err + } if err := ValidateKnowledgeConnectors(c.Knowledge.Connectors); err != nil { return err } diff --git a/src/pkg/convergence/mutation/boundary.go b/src/pkg/convergence/mutation/boundary.go index 08dea35b86..48e2ff1402 100644 --- a/src/pkg/convergence/mutation/boundary.go +++ b/src/pkg/convergence/mutation/boundary.go @@ -40,6 +40,34 @@ type Boundary struct { // leaves the unresolved state: applied, already applied, or reconciled. OnResolved func(logicalID string) StuckAfter time.Duration + // UnresolvedTTL bounds, per effect kind, how long a journaled operation + // may stay Planned/Unknown while Execute keeps refusing retries of it. + // Past it the refused call expires the operation to NotApplied (see + // ExpireUnresolved) and reports it resolved, so the stuck alert stops and + // the caller's NEXT attempt — which runs its own duplicate check first — + // may proceed. The refused call itself never runs the effect. Nil uses + // DefaultUnresolvedTTLs; a missing or non-positive entry keeps refusing + // until an explicit Reconcile. + UnresolvedTTL map[string]time.Duration +} + +// DefaultUnresolvedTTLs returns the per-kind unresolved TTLs a Boundary uses +// when UnresolvedTTL is nil. Only issue creates are listed: every issue +// create caller searches for an existing twin before it reaches Execute, so +// an Unknown create that is still refused an hour later was never found and +// a fresh attempt cannot duplicate a visible issue. Without this an advisory +// issue create left Unknown by an ambiguous response stayed refused and +// reported stuck forever (hivecommons/hive#17693). +func DefaultUnresolvedTTLs() map[string]time.Duration { + return map[string]time.Duration{effects.KindIssueCreate: effects.DefaultUnresolvedTTL} +} + +func (b *Boundary) unresolvedTTL(kind string) time.Duration { + ttls := b.UnresolvedTTL + if ttls == nil { + ttls = DefaultUnresolvedTTLs() + } + return ttls[kind] } // ExecuteJournaled preserves legacy Execute's off-mode passthrough while @@ -190,6 +218,10 @@ func (b *Boundary) Execute(ctx context.Context, claim effects.Claim, effect func return out, fmt.Errorf("%w: %v", effects.ErrDenied, execErr) } if errors.Is(execErr, ErrNeedsReconciliation) { + if b.expireRefused(ctx, claim) { + b.noteResolved(logicalID) + return out, fmt.Errorf("%w: %w: %w", effects.ErrNeedsReconciliation, execErr, effects.ErrUnresolvedExpired) + } b.noteStuck(executor, logicalID) return out, fmt.Errorf("%w: %w", effects.ErrNeedsReconciliation, execErr) } @@ -223,6 +255,25 @@ func (b *Boundary) noteStuck(executor Executor, logicalID string) { b.OnStuck(op, age) } +// expireRefused expires claim's refused operation to NotApplied once it has +// been unresolved for longer than its kind's UnresolvedTTL. It never runs the +// effect: the retry is left to the caller's next attempt. +func (b *Boundary) expireRefused(ctx context.Context, claim effects.Claim) bool { + ttl := b.unresolvedTTL(claim.Kind) + if ttl <= 0 { + return false + } + expired, err := b.ExpireUnresolved(ctx, claim, ttl) + if err != nil { + if b.Logger != nil { + b.Logger.Warn("expiring unresolved external mutation failed", + "repo", claim.Repo, "kind", claim.Kind, "target", claim.Target, "error", err) + } + return false + } + return expired +} + func (b *Boundary) noteResolved(logicalID string) { if b.OnResolved != nil && logicalID != "" { b.OnResolved(logicalID) diff --git a/src/pkg/convergence/mutation/boundary_unresolved_ttl_test.go b/src/pkg/convergence/mutation/boundary_unresolved_ttl_test.go new file mode 100644 index 0000000000..7ae2e659aa --- /dev/null +++ b/src/pkg/convergence/mutation/boundary_unresolved_ttl_test.go @@ -0,0 +1,156 @@ +package mutation + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/effects" +) + +// hivecommons/hive#17693: an issue create left Unknown by an ambiguous +// response, whose caller only ever calls Execute (no Reconcile/Expire of its +// own), was refused and reported stuck every cycle for 15h. Past the kind's +// unresolved TTL the refused call must expire the op to NotApplied, clear the +// stuck alert and — without running the effect in that call — let the +// caller's next attempt create. +func TestBoundaryExpiresRefusedUnknownIssueCreatePastTTL(t *testing.T) { + b := testBoundary(t, "enforce") + now := time.Unix(100, 0) + b.Executor.Now = func() time.Time { return now } + stuck := 0 + var resolved []string + b.OnStuck = func(Operation, time.Duration) { stuck++ } + b.OnResolved = func(id string) { resolved = append(resolved, id) } + claim := effects.Claim{Repo: "acme/widget", Kind: effects.KindIssueCreate, Target: "advisory"} + id := boundaryEffect(claim).LogicalID() + mustNotRun := func(context.Context) (effects.Result, error) { + t.Fatal("effect must not run on a refused attempt") + return effects.Result{}, nil + } + + ambiguous := errors.New("502 Bad Gateway") + if _, err := b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + return effects.Result{}, ambiguous + }); !errors.Is(err, ambiguous) { + t.Fatalf("first execute err = %v", err) + } + + // Inside the TTL: refused and reported stuck, still Unknown. + now = now.Add(DefaultStuckAfter) + _, err := b.Execute(context.Background(), claim, mustNotRun) + if !errors.Is(err, effects.ErrNeedsReconciliation) || errors.Is(err, effects.ErrUnresolvedExpired) { + t.Fatalf("inside TTL err = %v, want a plain refusal", err) + } + if op, _ := b.Executor.Journal.Get(id); op.Status != StatusUnknown || stuck != 1 { + t.Fatalf("inside TTL: status=%s stuck=%d, want Unknown and 1", op.Status, stuck) + } + + // Past the TTL: still refused this call, but expired and resolved. + now = now.Add(effects.DefaultUnresolvedTTL) + _, err = b.Execute(context.Background(), claim, mustNotRun) + if !errors.Is(err, effects.ErrNeedsReconciliation) || !errors.Is(err, effects.ErrUnresolvedExpired) { + t.Fatalf("past TTL err = %v, want refusal carrying ErrUnresolvedExpired", err) + } + if op, _ := b.Executor.Journal.Get(id); op.Status != StatusNotApplied { + t.Fatalf("past TTL status = %s, want NotApplied", op.Status) + } + if stuck != 1 { + t.Fatalf("stuck = %d after expiry, want no further report", stuck) + } + if len(resolved) != 1 || resolved[0] != id { + t.Fatalf("resolved = %v, want [%s]", resolved, id) + } + + // The next attempt is a fresh one and runs the effect. + ran := false + res, err := b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + ran = true + return effects.Result{Provenance: "https://github.example/acme/widget/issues/9"}, nil + }) + if err != nil || !ran || res.Provenance == "" { + t.Fatalf("retry = (%+v, %v, ran=%v), want the create to run", res, err, ran) + } + if stuck != 1 { + t.Fatalf("stuck = %d after successful retry", stuck) + } +} + +// Reconciliation still wins inside the TTL: an issue found externally +// resolves the op Applied (replayed without creating), one definitively +// absent releases a retry. +func TestBoundaryReconcileResolvesUnknownIssueCreate(t *testing.T) { + for _, tc := range []struct { + name string + state effects.ExternalState + status string + runs bool + }{ + {"found", effects.ExternalState{Applied: true, Provenance: "https://github.example/acme/widget/issues/7"}, StatusApplied, false}, + {"absent", effects.ExternalState{Applied: false}, StatusNotApplied, true}, + } { + t.Run(tc.name, func(t *testing.T) { + b := testBoundary(t, "enforce") + claim := effects.Claim{Repo: "acme/widget", Kind: effects.KindIssueCreate, Target: "finding"} + _, _ = b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + return effects.Result{}, errors.New("connection reset") + }) + if err := b.Reconcile(context.Background(), claim, tc.state); err != nil { + t.Fatal(err) + } + if op, _ := b.Executor.Journal.Get(boundaryEffect(claim).LogicalID()); op.Status != tc.status { + t.Fatalf("status = %s, want %s", op.Status, tc.status) + } + ran := false + res, err := b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + ran = true + return effects.Result{Provenance: "https://github.example/acme/widget/issues/8"}, nil + }) + if err != nil || ran != tc.runs { + t.Fatalf("after reconcile: err=%v ran=%v, want ran=%v", err, ran, tc.runs) + } + if !tc.runs && res.Provenance != tc.state.Provenance { + t.Fatalf("replay provenance = %q, want %q", res.Provenance, tc.state.Provenance) + } + }) + } +} + +// Kinds without an unresolved TTL keep refusing (and reporting stuck) until +// an explicit Reconcile, however old the entry; an explicit empty map +// disables the default for issue creates too. +func TestBoundaryUnresolvedTTLOnlyForConfiguredKinds(t *testing.T) { + for _, tc := range []struct { + name string + kind string + ttls map[string]time.Duration + }{ + {"merge default", effects.KindPullRequestMerge, nil}, + {"issue create disabled", effects.KindIssueCreate, map[string]time.Duration{}}, + } { + t.Run(tc.name, func(t *testing.T) { + b := testBoundary(t, "enforce") + now := time.Unix(100, 0) + b.Executor.Now = func() time.Time { return now } + b.UnresolvedTTL = tc.ttls + stuck := 0 + b.OnStuck = func(Operation, time.Duration) { stuck++ } + claim := effects.Claim{Repo: "acme/widget", Kind: tc.kind, Target: "7"} + _, _ = b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + return effects.Result{}, errors.New("timeout") + }) + now = now.Add(48 * time.Hour) + _, err := b.Execute(context.Background(), claim, func(context.Context) (effects.Result, error) { + t.Fatal("effect must not run") + return effects.Result{}, nil + }) + if !errors.Is(err, effects.ErrNeedsReconciliation) || errors.Is(err, effects.ErrUnresolvedExpired) { + t.Fatalf("err = %v, want a plain refusal", err) + } + if op, _ := b.Executor.Journal.Get(boundaryEffect(claim).LogicalID()); op.Status != StatusUnknown || stuck != 1 { + t.Fatalf("status=%s stuck=%d, want Unknown and 1", op.Status, stuck) + } + }) + } +} diff --git a/src/pkg/dashboard/agent_sandbox_warnings.test.js b/src/pkg/dashboard/agent_sandbox_warnings.test.js new file mode 100644 index 0000000000..4c5b8dc18d --- /dev/null +++ b/src/pkg/dashboard/agent_sandbox_warnings.test.js @@ -0,0 +1,29 @@ +// Agent card renders the per-agent sandbox_warnings preflight findings. +const test = require('node:test'); +const assert = require('node:assert'); +const fs = require('node:fs'); +const path = require('node:path'); + +const src = fs.readFileSync(path.join(__dirname, 'static-src', 'js', 'agent-cards.js'), 'utf8'); +const m = src.match(/function agentSandboxWarningsHtml\(a\) \{[\s\S]*?\n \}\n/); +assert.ok(m, 'agentSandboxWarningsHtml not found'); +const esc = s => String(s).replace(/&/g, '&').replace(/</g, '<').replace(/"/g, '"'); +const agentSandboxWarningsHtml = new Function('esc', m[0] + '; return agentSandboxWarningsHtml;')(esc); + +test('renders one agent-error line per warning', () => { + const html = agentSandboxWarningsHtml({ sandbox_warnings: ['operator missing', 'no <gvisor>'] }); + assert.strictEqual((html.match(/class="agent-error"/g) || []).length, 2); + assert.ok(html.includes('⚠ operator missing')); + assert.ok(html.includes('no <gvisor>')); +}); + +test('renders nothing without warnings', () => { + assert.strictEqual(agentSandboxWarningsHtml({}), ''); + assert.strictEqual(agentSandboxWarningsHtml({ sandbox_warnings: [] }), ''); + assert.strictEqual(agentSandboxWarningsHtml(null), ''); +}); + +test('shipped index.html carries the card hook', () => { + const html = fs.readFileSync(path.join(__dirname, 'static', 'index.html'), 'utf8'); + assert.ok(html.includes('${agentSandboxWarningsHtml(a)}')); +}); diff --git a/src/pkg/dashboard/agent_sandbox_warnings_test.go b/src/pkg/dashboard/agent_sandbox_warnings_test.go new file mode 100644 index 0000000000..2bb774ca7e --- /dev/null +++ b/src/pkg/dashboard/agent_sandbox_warnings_test.go @@ -0,0 +1,58 @@ +package dashboard + +import ( + "reflect" + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/governor" +) + +func TestAgentStatusSandboxWarningsByRuntime(t *testing.T) { + config.StoreSandboxGateReport(config.SandboxGateReport{ + OpenShell: []string{"openshell gateway unreachable"}, + Substrate: []string{"substrate operator not found"}, + }) + t.Cleanup(func() { config.StoreSandboxGateReport(config.SandboxGateReport{}) }) + + enabled := true + agent := func(runtime string, sandboxed bool) config.AgentConfig { + a := config.AgentConfig{Backend: "claude", Enabled: true} + if sandboxed { + a.Sandbox = &config.AgentSandboxOverride{Enabled: &enabled, Runtime: runtime} + } + return a + } + cfg := &config.Config{ + AgentRuntimeConfig: config.AgentRuntimeConfig{ + AgentSandbox: config.AgentSandboxConfig{Enabled: true, Image: "img"}, + Agents: map[string]config.AgentConfig{ + "sub": agent("substrate", true), + "osh": agent("openshell", true), + "job": agent("job", true), + "none": agent("", false), + }, + }, + Governor: config.GovernorConfig{Modes: map[string]config.ModeConfig{}}, + } + statuses := map[string]*crux.Process{} + for n, a := range cfg.Agents { + statuses[n] = &crux.Process{Config: a} + } + payload := BuildAgentOnlyStatus(governor.State{Mode: "IDLE"}, statuses, cfg) + + want := map[string][]string{ + "sub": {"substrate operator not found"}, + "osh": {"openshell gateway unreachable"}, + "job": nil, + "none": nil, + } + got := map[string][]string{} + for _, a := range payload.Agents { + got[a.Name] = a.SandboxWarnings + } + if !reflect.DeepEqual(got, want) { + t.Fatalf("sandbox warnings by agent = %#v, want %#v", got, want) + } +} diff --git a/src/pkg/dashboard/api.go b/src/pkg/dashboard/api.go index 1ef02621b6..7c9d849cd7 100644 --- a/src/pkg/dashboard/api.go +++ b/src/pkg/dashboard/api.go @@ -583,6 +583,8 @@ func (s *Server) RegisterAPI(deps *Dependencies) { s.mux.HandleFunc("GET /api/hotspots", s.handleHotspots) // Knowledge connector status + manual sync (#11069, api_knowledge_connectors.go). s.registerKnowledgeConnectorRoutes() + s.registerKnowledgeConnectorAuthRoutes() + s.registerInboxRoutes() s.mux.HandleFunc("POST /api/system-alerts/dismiss", s.handleSystemAlertDismiss) // Per-agent Variables tab (#11458, api_variables.go). s.registerAgentVariableRoutes() diff --git a/src/pkg/dashboard/api_agent_config.go b/src/pkg/dashboard/api_agent_config.go index eda51ca22e..3fb61fba92 100644 --- a/src/pkg/dashboard/api_agent_config.go +++ b/src/pkg/dashboard/api_agent_config.go @@ -207,6 +207,7 @@ func (s *Server) handleAgentConfigGet(w http.ResponseWriter, r *http.Request) { "jevMode": agentCfg.JevMode, "jevReady": s.deps.Config.JevReady(), "explainMode": agentCfg.ExplainMode, + "sandboxSubstrate": agentSandboxSubstrateOrEmpty(agentCfg), "sandboxEnabled": agentCfg.Sandbox != nil && agentCfg.Sandbox.Enabled != nil && *agentCfg.Sandbox.Enabled, "sandboxEffective": agentCfg.SandboxEnabled(s.deps.Config.AgentSandbox), "replicas": agentCfg.Replicas, @@ -765,6 +766,27 @@ func (s *Server) handleAgentConfigGeneral(w http.ResponseWriter, r *http.Request agentCfg.Sandbox.Enabled = &bb } } + if v, ok := body["sandboxSubstrate"]; ok { + raw, err := json.Marshal(v) + var in config.SandboxSubstrateConfig + if err == nil { + err = json.Unmarshal(raw, &in) + } + var sub *config.SandboxSubstrateConfig + if err == nil { + sub, err = sanitizeSubstrateConfig(in) + } + if err != nil { + jsonError(w, "invalid sandboxSubstrate: "+err.Error(), http.StatusBadRequest) + return + } + if sub != nil && agentCfg.Sandbox == nil { + agentCfg.Sandbox = &config.AgentSandboxOverride{} + } + if agentCfg.Sandbox != nil { + agentCfg.Sandbox.Substrate = sub + } + } if v, ok := body["cavemanMode"]; ok { if s, ok := v.(string); ok { s = sanitizeString(s) @@ -2081,3 +2103,10 @@ func (s *Server) getAgentHooks(name string) map[string][]any { } return map[string][]any{"preKick": {}, "postIdle": {}} } + +func agentSandboxSubstrateOrEmpty(a config.AgentConfig) config.SandboxSubstrateConfig { + if a.Sandbox == nil { + return config.SandboxSubstrateConfig{} + } + return substrateOrEmpty(a.Sandbox.Substrate) +} diff --git a/src/pkg/dashboard/api_benchmarks.go b/src/pkg/dashboard/api_benchmarks.go index ae05bddc3c..c11c0608c0 100644 --- a/src/pkg/dashboard/api_benchmarks.go +++ b/src/pkg/dashboard/api_benchmarks.go @@ -147,6 +147,7 @@ func (s *Server) benchmarksOptions(ctx context.Context, since time.Time) benchma RepoConsent: cfg.Benchmarks.RepoConsentEnabled(), RepoPublic: func(repo string) bool { return s.benchmarksRepoPublic(ctx, repo) }, SandboxRuntime: cfg.BenchmarkSandboxRuntime, + CostLookup: benchmarksCostLookup(s.deps.LaunchCost), } } diff --git a/src/pkg/dashboard/api_governor.go b/src/pkg/dashboard/api_governor.go index 5441771847..cd86642b56 100644 --- a/src/pkg/dashboard/api_governor.go +++ b/src/pkg/dashboard/api_governor.go @@ -3369,20 +3369,6 @@ func firstRepoPathSegment(ref string) string { // a repo the operator is about to add, and — when it cannot — returns the // per-forge App install/authorize URL so the Repos tab can guide the user to // grant access before the repo is finally accepted (requirement #3). -// -// The check today probes at ORG granularity: DiscoverInstallationID(org) returns -// ErrNoInstallationForOrg when the App is not installed on the repo's org at all, -// which is the common "new org the App was never installed on" case and the one -// that hard-blocks reads and writes. On a positive result we report ok:true. -// -// TODO(repo-access-probe): tighten to REPO granularity. An App installed on the -// org "Only select repositories" may still lack THIS repo (see #2353's -// write-forbidden case). The deeper probe should mint an installation token and -// GET /repos/{org}/{repo} (s.deps.GHClient.GetRepo) — a 404/403 there means the -// installation exists but this repo is not in its selected set, which needs the -// same "manage installation" URL to add the repo to the selection. That probe is -// scoped out of this change to keep it focused; the org-level check plus the -// install workflow already covers the "App not installed on the org" footgun. func (s *Server) handleGovernorRepoCheckAccess(w http.ResponseWriter, r *http.Request) { var body struct { Repo string `json:"repo"` @@ -3423,6 +3409,18 @@ func (s *Server) handleGovernorRepoCheckAccess(w http.ResponseWriter, r *http.Re jsonError(w, "cannot determine the org for this repo — use org/repo format", http.StatusBadRequest) return } + if len(parts) == 1 { + parts = []string{org, parts[0]} + } + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + jsonError(w, "cannot determine the repository — use org/repo format", http.StatusBadRequest) + return + } + repo := strings.TrimSuffix(parts[1], ".git") + if repo == "" { + jsonError(w, "cannot determine the repository — use org/repo format", http.StatusBadRequest) + return + } // An App-less hive (advisory-only, no private key) cannot and need not probe // installation access: it never writes. Treat it as "no access check needed" @@ -3436,13 +3434,13 @@ func (s *Server) handleGovernorRepoCheckAccess(w http.ResponseWriter, r *http.Re if ctx == nil { ctx = r.Context() } - if _, err := s.deps.GHAppAuth.DiscoverInstallationID(ctx, org); err != nil { - // Not installed on this org (or discovery failed). Surface the correct - // per-forge install URL and arm the existing pending-install mechanism so - // the banner/recheck plumbing the App-required flow already uses drives - // this to completion. AppInstallURL() returns "" for a GHE host whose App - // slug was never configured — the UI renders that as a config message - // rather than a dead link. + installationID, err := s.deps.GHAppAuth.DiscoverInstallationID(ctx, org) + if err != nil { + if !errors.Is(err, github.ErrNoInstallationForOrg) { + s.logger.Warn("repo add: installation discovery failed", "org", org, "error", err) + jsonError(w, "could not check GitHub App installation access; please retry", http.StatusServiceUnavailable) + return + } installURL := "" if s.deps.Config != nil { installURL = s.deps.Config.GitHub.AppInstallURL() @@ -3460,6 +3458,41 @@ func (s *Server) handleGovernorRepoCheckAccess(w http.ResponseWriter, r *http.Re }) return } + coverage, err := s.deps.GHAppAuth.InstallationCoverageForID(ctx, installationID) + if err != nil { + s.logger.Warn("repo add: installation repository coverage failed", "org", org, "repo", repo, "error", err) + jsonError(w, "could not check GitHub App repository access; please retry", http.StatusServiceUnavailable) + return + } + if _, found := coverage.Repos[strings.ToLower(org+"/"+repo)]; !found { + if coverage.Truncated { + covered, err := s.deps.GHAppAuth.RepositoryCoveredByInstallation(ctx, installationID, org, repo) + if err != nil { + s.logger.Warn("repo add: repository installation probe failed", "org", org, "repo", repo, "error", err) + jsonError(w, "could not check GitHub App repository access; please retry", http.StatusServiceUnavailable) + return + } + if covered { + okResponse(w, map[string]string{"status": "ok", "org": org}) + return + } + } + installURL := (github.AppAuthDiagnosis{ + ExpectedAccount: org, + InstallationID: installationID, + APIURL: s.deps.Config.GitHub.ResolvedAPIURL(), + }).InstallationSettingsURL() + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusConflict) + _ = json.NewEncoder(w).Encode(map[string]interface{}{ + "ok": false, + "needsInstall": true, + "org": org, + "installUrl": installURL, + "error": fmt.Sprintf("the Hive GitHub App installation cannot access %s/%s. Add the repository to the installation's repository access, then re-check.", org, repo), + }) + return + } okResponse(w, map[string]string{"status": "ok", "org": org}) } diff --git a/src/pkg/dashboard/api_governor_security.go b/src/pkg/dashboard/api_governor_security.go index 36200d4ef6..a38f3c9539 100644 --- a/src/pkg/dashboard/api_governor_security.go +++ b/src/pkg/dashboard/api_governor_security.go @@ -1,8 +1,10 @@ package dashboard import ( + "errors" "net/http" "os" + "reflect" "sort" "strings" @@ -40,27 +42,28 @@ func (s *Server) handleGovernorSecurity(w http.ResponseWriter, r *http.Request) IntentAlignmentModel *string `json:"intentAlignmentModel"` // Intent classifier lists. An empty list restores the built-in // defaults (nil), matching how the intent package treats them. - IntentTestPathPatterns *[]string `json:"intentTestPathPatterns"` - IntentDocsPathPatterns *[]string `json:"intentDocsPathPatterns"` - IntentGuardrailPathPatterns *[]string `json:"intentGuardrailPathPatterns"` - IntentFeatureSignals *[]string `json:"intentFeatureSignals"` - ReviewRequire *bool `json:"reviewRequireApproval"` - ReviewFanOut *bool `json:"reviewFanOut"` - ReviewMaxParallel *int `json:"reviewMaxParallelReviews"` - ReviewReviewerAgents *[]string `json:"reviewReviewerAgents"` - ReviewFixerAgent *string `json:"reviewFixerAgent"` - AgentSandboxEnabled *bool `json:"agentSandboxEnabled"` - SandboxImage *string `json:"agentSandboxImage"` - SandboxEnvAllowlist *[]string `json:"agentSandboxEnvAllowlist"` - SandboxNetworkMode *string `json:"agentSandboxNetworkMode"` - SandboxTimeoutS *int `json:"agentSandboxTimeoutS"` - SandboxWorkspaceDir *string `json:"agentSandboxWorkspaceDir"` - SandboxRuntime *string `json:"agentSandboxRuntime"` - SandboxJobServiceAccount *string `json:"agentSandboxJobServiceAccount"` - SandboxJobSchedulerName *string `json:"agentSandboxJobSchedulerName"` - SandboxJobTTLSeconds *int `json:"agentSandboxJobTtlSeconds"` - SandboxJobWorkspaceClaim *string `json:"agentSandboxJobWorkspaceClaim"` - SandboxJobClaimMount *string `json:"agentSandboxJobWorkspaceClaimMount"` + IntentTestPathPatterns *[]string `json:"intentTestPathPatterns"` + IntentDocsPathPatterns *[]string `json:"intentDocsPathPatterns"` + IntentGuardrailPathPatterns *[]string `json:"intentGuardrailPathPatterns"` + IntentFeatureSignals *[]string `json:"intentFeatureSignals"` + ReviewRequire *bool `json:"reviewRequireApproval"` + ReviewFanOut *bool `json:"reviewFanOut"` + ReviewMaxParallel *int `json:"reviewMaxParallelReviews"` + ReviewReviewerAgents *[]string `json:"reviewReviewerAgents"` + ReviewFixerAgent *string `json:"reviewFixerAgent"` + AgentSandboxEnabled *bool `json:"agentSandboxEnabled"` + SandboxImage *string `json:"agentSandboxImage"` + SandboxEnvAllowlist *[]string `json:"agentSandboxEnvAllowlist"` + SandboxNetworkMode *string `json:"agentSandboxNetworkMode"` + SandboxTimeoutS *int `json:"agentSandboxTimeoutS"` + SandboxWorkspaceDir *string `json:"agentSandboxWorkspaceDir"` + SandboxRuntime *string `json:"agentSandboxRuntime"` + SandboxJobServiceAccount *string `json:"agentSandboxJobServiceAccount"` + SandboxJobSchedulerName *string `json:"agentSandboxJobSchedulerName"` + SandboxJobTTLSeconds *int `json:"agentSandboxJobTtlSeconds"` + SandboxJobWorkspaceClaim *string `json:"agentSandboxJobWorkspaceClaim"` + SandboxJobClaimMount *string `json:"agentSandboxJobWorkspaceClaimMount"` + SandboxSubstrate *config.SandboxSubstrateConfig `json:"agentSandboxSubstrate"` // Sentinel (suspicious-activity alerts) is pointer-typed per field // so an absent key means "unchanged" — a save that only toggled a // behavior must not wipe the sensitive-path list. @@ -325,6 +328,14 @@ func (s *Server) handleGovernorSecurity(w http.ResponseWriter, r *http.Request) job.TTLSeconds = *body.SandboxJobTTLSeconds } } + if body.SandboxSubstrate != nil { + sub, err := sanitizeSubstrateConfig(*body.SandboxSubstrate) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + cfg.AgentSandbox.Substrate = sub + } if nextSentinel != nil { cfg.Sentinel = *nextSentinel } @@ -488,6 +499,7 @@ func securitySectionResponse(cfg *config.Config) map[string]interface{} { "agentSandboxJobTtlSeconds": sandboxJobOrEmpty(cfg).TTLSeconds, "agentSandboxJobWorkspaceClaim": sandboxJobOrEmpty(cfg).WorkspaceClaim, "agentSandboxJobWorkspaceClaimMount": sandboxJobOrEmpty(cfg).WorkspaceClaimMount, + "agentSandboxSubstrate": substrateOrEmpty(cfg.AgentSandbox.Substrate), "sandboxedAgents": sandboxed, "totalAgents": len(cfg.Agents), "sandboxWarnings": sandboxWarnings, @@ -597,3 +609,50 @@ func sandboxJobOrEmpty(cfg *config.Config) config.SandboxJobConfig { } return *cfg.AgentSandbox.Job } + +func substrateOrEmpty(sub *config.SandboxSubstrateConfig) config.SandboxSubstrateConfig { + if sub == nil { + return config.SandboxSubstrateConfig{} + } + return *sub +} + +// sanitizeSubstrateConfig trims and validates a Substrate block from the +// Settings dialog. An all-empty block returns nil so the YAML key is dropped. +func sanitizeSubstrateConfig(in config.SandboxSubstrateConfig) (*config.SandboxSubstrateConfig, error) { + if in.TimeoutS < 0 || in.BridgePort < 0 || in.CPUMillicores < 0 || in.MemoryBytes < 0 { + return nil, errors.New("substrate numeric settings must be >= 0") + } + clean := func(s string) string { return sanitizeString(strings.TrimSpace(s)) } + out := config.SandboxSubstrateConfig{ + Endpoint: clean(in.Endpoint), + Atespace: clean(in.Atespace), + TokenEnv: clean(in.TokenEnv), + TokenFile: clean(in.TokenFile), + CAFile: clean(in.CAFile), + InsecureTLS: in.InsecureTLS, + TimeoutS: in.TimeoutS, + RouterURL: clean(in.RouterURL), + RouterClientBundle: clean(in.RouterClientBundle), + RouterCAFile: clean(in.RouterCAFile), + BridgePort: in.BridgePort, + Image: clean(in.Image), + BridgeCommand: sanitizeStringSlice(in.BridgeCommand), + CPUMillicores: in.CPUMillicores, + MemoryBytes: in.MemoryBytes, + WorkerSelector: sanitizedHeaderMap(in.WorkerSelector), + SnapshotOnIdle: in.SnapshotOnIdle, + AllowHosts: sanitizeStringSlice(in.AllowHosts), + PolicyProfile: clean(in.PolicyProfile), + } + if len(out.BridgeCommand) == 0 { + out.BridgeCommand = nil + } + if len(out.AllowHosts) == 0 { + out.AllowHosts = nil + } + if reflect.DeepEqual(out, config.SandboxSubstrateConfig{}) { + return nil, nil + } + return &out, nil +} diff --git a/src/pkg/dashboard/api_inbox.go b/src/pkg/dashboard/api_inbox.go new file mode 100644 index 0000000000..67dfba6281 --- /dev/null +++ b/src/pkg/dashboard/api_inbox.go @@ -0,0 +1,390 @@ +package dashboard + +import ( + "context" + "errors" + "net/http" + "os" + "os/exec" + "strconv" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/inbox" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// inboxListTimeout bounds one Inbox listing (reads connector facts). +const inboxListTimeout = 20 * time.Second + +// Inbox (#voice-sources): source-agnostic triage of voice-recorder items +// that knowledge connectors ingest, with routes to spek RFCs and dibs. +func (s *Server) registerInboxRoutes() { + s.mux.HandleFunc("GET /api/inbox", s.handleInboxList) + s.mux.HandleFunc("POST /api/inbox/route", s.handleInboxRoute) + s.mux.HandleFunc("POST /api/inbox/dismiss", s.handleInboxDismiss) + s.mux.HandleFunc("GET /api/inbox/settings", s.handleInboxSettingsGet) + s.mux.HandleFunc("POST /api/inbox/settings", s.handleInboxSettingsPost) + s.mux.HandleFunc("POST /api/knowledge/connectors/{name}/auth/complete", s.handleKnowledgeConnectorAuthComplete) +} + +// InboxSourceStatus is one configured source as the Inbox header shows it. +type InboxSourceStatus struct { + Type string `json:"type"` + Connector string `json:"connector,omitempty"` + // Configured is true when a knowledge.connectors entry of this type exists. + Configured bool `json:"configured"` + // Connected mirrors the connector's AuthState when it signs in interactively. + Connected bool `json:"connected"` + Account string `json:"account,omitempty"` +} + +// InboxListResponse is GET /api/inbox. +type InboxListResponse struct { + Items []inbox.Item `json:"items"` + Unprocessed int `json:"unprocessed"` + Sources []InboxSourceStatus `json:"sources"` + // Repos are the hive's configured repos for the RFC repo picker. + Repos []string `json:"repos"` + Settings inbox.Settings `json:"settings"` + Error string `json:"error,omitempty"` +} + +func (s *Server) inboxService(w http.ResponseWriter) (*inbox.Service, bool) { + if s == nil || s.deps == nil || s.deps.Inbox == nil { + jsonError(w, "inbox is not available: configure a voice knowledge connector (plaud, wisprflow)", http.StatusNotFound) + return nil, false + } + return s.deps.Inbox, true +} + +func (s *Server) handleInboxList(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithTimeout(r.Context(), inboxListTimeout) + defer cancel() + if s == nil || s.deps == nil || s.deps.Inbox == nil { + // No service (older boot path / tests): still a valid, empty inbox — + // the panel must render its empty state, not spin on a 404. + resp := InboxListResponse{Items: []inbox.Item{}, Sources: []InboxSourceStatus{}, Repos: s.inboxRepos(), Settings: inbox.Settings{Rules: []config.InboxRule{}}} + if s != nil && s.deps != nil && s.deps.Config != nil { + resp.Sources = s.inboxSources(ctx, s.deps.Config.Inbox.EffectiveSources()) + } + jsonResponse(w, resp) + return + } + svc := s.deps.Inbox + items, err := svc.Items(ctx) + resp := InboxListResponse{ + Items: items, + Unprocessed: inbox.Unprocessed(items), + Sources: s.inboxSources(ctx, svc.Config().EffectiveSources()), + Repos: s.inboxRepos(), + Settings: svc.Settings(), + } + if resp.Items == nil { + resp.Items = []inbox.Item{} + } + if err != nil { + resp.Error = err.Error() + } + jsonResponse(w, resp) +} + +// inboxSources pairs each source type with its connector's cached auth state. +func (s *Server) inboxSources(ctx context.Context, types []string) []InboxSourceStatus { + out := make([]InboxSourceStatus, 0, len(types)) + rt := s.knowledgeConnectors() + var statuses []connector.Status + if rt != nil { + statuses = rt.Statuses() + } + lookup, _ := rt.(knowledgeConnectorLookup) + for _, typ := range types { + st := InboxSourceStatus{Type: typ} + for _, cs := range statuses { + if cs.Type != typ { + continue + } + st.Configured, st.Connector = true, cs.Name + if lookup != nil { + if c, ok := lookup.Connector(cs.Name); ok { + if auth, ok := c.(connector.Authenticator); ok { + a := auth.AuthStatus(ctx, false) + st.Connected, st.Account = a.Connected, a.Account + } + } + } + break + } + out = append(out, st) + } + return out +} + +func (s *Server) inboxRepos() []string { + if s == nil || s.deps == nil || s.deps.Config == nil { + return []string{} + } + repos := append([]string(nil), s.deps.Config.Project.Repos...) + if def := s.deps.Config.Inbox.Spek.Repo; def != "" { + found := false + for _, r := range repos { + if r == def { + found = true + } + } + if !found { + repos = append([]string{def}, repos...) + } + } + if repos == nil { + repos = []string{} + } + return repos +} + +type inboxRouteRequest struct { + Source string `json:"source"` + ItemID string `json:"item_id"` + Destination string `json:"destination"` + Repo string `json:"repo,omitempty"` +} + +// InboxRouteResponse is POST /api/inbox/route. +type InboxRouteResponse struct { + Source string `json:"source"` + ItemID string `json:"item_id"` + Destination string `json:"destination"` + State inbox.RouteState `json:"state"` +} + +// handleInboxRoute (owner) queues one item for one destination; progress is +// read back from GET /api/inbox. +func (s *Server) handleInboxRoute(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + svc, ok := s.inboxService(w) + if !ok { + return + } + var req inboxRouteRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, "invalid request body", http.StatusBadRequest) + return + } + req.Source, req.ItemID, req.Destination, req.Repo = strings.TrimSpace(req.Source), strings.TrimSpace(req.ItemID), strings.TrimSpace(req.Destination), strings.TrimSpace(req.Repo) + if req.Source == "" || req.ItemID == "" || req.Destination == "" { + jsonError(w, "source, item_id and destination are required", http.StatusBadRequest) + return + } + if req.Repo != "" && !inboxRepoOK(req.Repo) { + jsonError(w, "repo must be owner/repo", http.StatusBadRequest) + return + } + ctx, cancel := context.WithTimeout(r.Context(), inboxListTimeout) + defer cancel() + it, found, err := svc.Find(ctx, req.Source, req.ItemID) + if !found { + msg := "unknown inbox item" + if err != nil { + msg += ": " + err.Error() + } + jsonError(w, msg, http.StatusNotFound) + return + } + st, err := svc.Route(ctx, it, inbox.Request{Destination: req.Destination, Repo: req.Repo}) + switch { + case errors.Is(err, inbox.ErrRouteBusy): + jsonError(w, "that route is already queued or running", http.StatusConflict) + return + case errors.Is(err, inbox.ErrUnknownDestination): + jsonError(w, err.Error(), http.StatusBadRequest) + return + case err != nil: + jsonError(w, "route failed: "+err.Error(), http.StatusInternalServerError) + return + } + s.auditFromRequest(r, "inbox_route", auditDetail("source", req.Source, "item", req.ItemID, "destination", req.Destination, "repo", req.Repo), "") + jsonStatusResponse(w, http.StatusAccepted, InboxRouteResponse{Source: req.Source, ItemID: req.ItemID, Destination: req.Destination, State: st}) +} + +func inboxRepoOK(repo string) bool { + owner, name, ok := strings.Cut(repo, "/") + return ok && owner != "" && name != "" && !strings.ContainsAny(repo, " \t\n") +} + +type inboxDismissRequest struct { + Items []struct { + Source string `json:"source"` + ItemID string `json:"item_id"` + } `json:"items"` + Undo bool `json:"undo,omitempty"` +} + +// InboxDismissResponse is POST /api/inbox/dismiss. +type InboxDismissResponse struct { + Dismissed int `json:"dismissed"` +} + +// handleInboxDismiss (owner) marks items "knowledge only" (bulk). +func (s *Server) handleInboxDismiss(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + svc, ok := s.inboxService(w) + if !ok { + return + } + var req inboxDismissRequest + if err := decodeBody(r, &req); err != nil || len(req.Items) == 0 { + jsonError(w, "items[] is required", http.StatusBadRequest) + return + } + n := 0 + for _, it := range req.Items { + if it.Source == "" || it.ItemID == "" { + continue + } + if err := svc.Dismiss(it.Source, it.ItemID, !req.Undo); err != nil { + jsonError(w, "dismiss failed: "+err.Error(), http.StatusInternalServerError) + return + } + n++ + } + s.auditFromRequest(r, "inbox_dismiss", auditDetail("count", strconv.Itoa(n)), "") + jsonResponse(w, InboxDismissResponse{Dismissed: n}) +} + +func (s *Server) handleInboxSettingsGet(w http.ResponseWriter, r *http.Request) { + svc, ok := s.inboxService(w) + if !ok { + return + } + jsonResponse(w, svc.Settings()) +} + +type inboxSettingsRequest struct { + Rules *[]config.InboxRule `json:"rules,omitempty"` + SpekRepo *string `json:"spek_repo,omitempty"` + DibsURL *string `json:"dibs_url,omitempty"` +} + +// handleInboxSettingsPost (owner) updates inbox.rules / inbox.spek.repo / +// inbox.dibs.url and persists hive.yaml. +func (s *Server) handleInboxSettingsPost(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + svc, ok := s.inboxService(w) + if !ok { + return + } + if s.deps.Config == nil { + jsonError(w, "config is not available", http.StatusServiceUnavailable) + return + } + var req inboxSettingsRequest + if err := decodeBody(r, &req); err != nil { + jsonError(w, "invalid request body", http.StatusBadRequest) + return + } + next := s.deps.Config.Inbox + if req.Rules != nil { + next.Rules = *req.Rules + } + if req.SpekRepo != nil { + next.Spek.Repo = strings.TrimSpace(*req.SpekRepo) + } + if req.DibsURL != nil { + next.Dibs.URL = strings.TrimRight(strings.TrimSpace(*req.DibsURL), "/") + } + if err := config.ValidateInbox(next); err != nil { + jsonError(w, err.Error(), http.StatusBadRequest) + return + } + s.deps.Config.Inbox = next + svc.UpdateConfig(next) + if err := s.saveConfig(); err != nil { + jsonError(w, "saving config: "+err.Error(), http.StatusInternalServerError) + return + } + s.auditFromRequest(r, "inbox_settings", auditDetail("rules", strconv.Itoa(len(next.Rules)), "spek_repo", next.Spek.Repo, "dibs_url", next.Dibs.URL), "") + jsonResponse(w, svc.Settings()) +} + +// handleKnowledgeConnectorAuthComplete (owner) finishes a sign-in whose +// OAuth callback cannot reach a hosted hive: the operator pastes either the +// redirect URL the browser landed on or the vendor token file contents. +func (s *Server) handleKnowledgeConnectorAuthComplete(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + name := r.PathValue("name") + auth, ok := s.knowledgeAuthenticator(w, name) + if !ok { + return + } + completer, ok := auth.(connector.AuthCompleter) + if !ok { + jsonError(w, "this connector does not support pasted sign-in completion", http.StatusBadRequest) + return + } + var comp connector.AuthCompletion + if err := decodeBody(r, &comp); err != nil { + jsonError(w, "invalid request body", http.StatusBadRequest) + return + } + ctx, cancel := knowledgeAuthContext(r) + defer cancel() + st, err := completer.CompleteAuth(ctx, comp) + if err != nil { + jsonError(w, err.Error(), http.StatusBadGateway) + return + } + mode := "redirect_url" + if comp.Tokens != "" { + mode = "tokens" + } + s.auditFromRequest(r, "knowledge_connector_auth_complete", auditDetail("connector", name, "mode", mode), "") + jsonResponse(w, st) +} + +// NewInboxDestinations builds the spek_rfc and dibs destinations from +// config; token minting and agent kicks come from the dashboard deps. +func NewInboxDestinations(cfg *config.Config, token func(ctx context.Context, repo string) (string, error), kick inbox.Kicker) []inbox.Destination { + binary := cfg.Inbox.Spek.Binary + if binary == "" { + binary = cfg.Runs.Spektacular.BinaryOrDefault() + } + spek := &inbox.SpekDestination{ + Binary: binary, + DefaultRepo: cfg.Inbox.Spek.Repo, + Token: token, + Agent: cfg.InboxSpekAgent(), + Kick: kick, + } + dibs := &inbox.DibsDestination{URL: cfg.Inbox.Dibs.URL} + if env := cfg.Inbox.Dibs.TokenEnv; env != "" { + dibs.Token = os.Getenv(env) + } + return []inbox.Destination{spek, dibs} +} + +// InboxReadiness returns the probes the settings view reports. +func InboxReadiness(cfg *config.Config) (spek func() bool, dibs func() bool) { + binary := cfg.Inbox.Spek.Binary + if binary == "" { + binary = cfg.Runs.Spektacular.BinaryOrDefault() + } + spek = func() bool { + if _, err := exec.LookPath(binary); err == nil { + return true + } + return cfg.InboxSpekAgent() != "" + } + dibs = func() bool { + return cfg.Inbox.Dibs.TokenEnv != "" && os.Getenv(cfg.Inbox.Dibs.TokenEnv) != "" + } + return spek, dibs +} diff --git a/src/pkg/dashboard/api_inbox_test.go b/src/pkg/dashboard/api_inbox_test.go new file mode 100644 index 0000000000..72f407b2b9 --- /dev/null +++ b/src/pkg/dashboard/api_inbox_test.go @@ -0,0 +1,226 @@ +package dashboard + +import ( + "context" + "encoding/json" + "errors" + "net/http" + "os" + "path/filepath" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/inbox" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +const inboxTestFact = `--- +title: RFC: Voice inbox +type: reference +layer: personal +status: active +source: plaud +connector: plaud +source_id: rec-1 +attr_plaud_file_id: rec-1 +attr_recorded_at: 2026-11-10T08:58:00Z +synthesized: 2026-11-10T08:58:00Z +--- + +## Summary (Plaud) + +- ship the inbox + +## Transcript + +Hello hive +` + +type inboxTestDest struct { + name string + got []inbox.Request + err error +} + +func (d *inboxTestDest) Name() string { return d.name } +func (d *inboxTestDest) Deliver(_ context.Context, _ inbox.Item, _ inbox.Content, req inbox.Request) (inbox.Result, error) { + d.got = append(d.got, req) + if d.err != nil { + return inbox.Result{}, d.err + } + return inbox.Result{URL: "https://github.com/o/r/issues/1", Children: []string{"https://github.com/o/r/issues/2"}}, nil +} + +func newInboxTestService(t *testing.T, cfg config.InboxConfig, dests ...inbox.Destination) *inbox.Service { + t.Helper() + base := t.TempDir() + dir := filepath.Join(base, "connectors", "personal") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "plaud-plaud-rec-1.md"), []byte(inboxTestFact), 0o644); err != nil { + t.Fatal(err) + } + src := &inbox.FactSource{ConnectorsDir: filepath.Join(base, "connectors"), SourceType: "plaud"} + svc := inbox.NewService(cfg, inbox.NewStore(filepath.Join(base, "inbox.json")), []inbox.Source{src}, dests, nil) + svc.Run = func(fn func()) { fn() } + return svc +} + +func decodeInboxList(t *testing.T, body []byte) InboxListResponse { + t.Helper() + var resp InboxListResponse + if err := json.Unmarshal(body, &resp); err != nil { + t.Fatalf("decode: %v (%s)", err, body) + } + return resp +} + +func TestInboxEndpoints(t *testing.T) { + s, deps := apiServer(t) + // No service: still 200 with an empty list so the panel renders its + // empty state instead of "Loading inbox…" forever (#17918). + rec0 := doGet(s, "/api/inbox") + if rec0.Code != http.StatusOK { + t.Fatalf("inbox without service = %d, want 200: %s", rec0.Code, rec0.Body.String()) + } + if empty := decodeInboxList(t, rec0.Body.Bytes()); empty.Items == nil || len(empty.Items) != 0 || empty.Sources == nil || empty.Settings.Rules == nil { + t.Fatalf("empty inbox = %s", rec0.Body.String()) + } + if rec := doGet(s, "/api/inbox/settings"); rec.Code != http.StatusNotFound { + t.Fatalf("settings without service = %d, want 404", rec.Code) + } + spek := &inboxTestDest{name: inbox.DestSpekRFC} + dibs := &inboxTestDest{name: inbox.DestDibs, err: errors.New("dibs down")} + deps.Config.Inbox = config.InboxConfig{Spek: config.InboxSpek{Repo: "myorg/rfcs"}} + deps.Inbox = newInboxTestService(t, deps.Config.Inbox, spek, dibs) + deps.KnowledgeConnectors = newTestConnectorSyncer(t, &authConn{state: connector.AuthState{Connected: true, Account: "andy@example.com"}}) + + rec := doGet(s, "/api/inbox") + if rec.Code != http.StatusOK { + t.Fatalf("list = %d: %s", rec.Code, rec.Body.String()) + } + resp := decodeInboxList(t, rec.Body.Bytes()) + if len(resp.Items) != 1 || resp.Items[0].ID != "rec-1" || resp.Items[0].FactSlug != "plaud-plaud-rec-1" || resp.Unprocessed != 1 { + t.Fatalf("list = %+v", resp) + } + if len(resp.Repos) != 2 || resp.Repos[0] != "myorg/rfcs" || resp.Repos[1] != "repo1" { + t.Fatalf("repos = %v", resp.Repos) + } + if len(resp.Sources) != 2 || resp.Sources[0].Type != "plaud" || resp.Sources[0].Configured || resp.Settings.SpekRepo != "myorg/rfcs" { + t.Fatalf("sources/settings = %+v %+v", resp.Sources, resp.Settings) + } + + if rec := doPostNoOwner(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "rec-1", "destination": "spek_rfc"}); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner route = %d, want 403", rec.Code) + } + if rec := doPost(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "nope", "destination": "spek_rfc"}); rec.Code != http.StatusNotFound { + t.Fatalf("unknown item = %d", rec.Code) + } + if rec := doPost(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "rec-1", "destination": "spek_rfc", "repo": "bad repo"}); rec.Code != http.StatusBadRequest { + t.Fatalf("bad repo = %d", rec.Code) + } + if rec := doPost(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "rec-1", "destination": "nowhere"}); rec.Code != http.StatusBadRequest { + t.Fatalf("unknown destination = %d: %s", rec.Code, rec.Body.String()) + } + rec = doPost(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "rec-1", "destination": "spek_rfc"}) + if rec.Code != http.StatusAccepted { + t.Fatalf("route = %d: %s", rec.Code, rec.Body.String()) + } + if len(spek.got) != 1 || spek.got[0].Repo != "myorg/rfcs" { + t.Fatalf("spek requests = %+v (default repo must apply)", spek.got) + } + resp = decodeInboxList(t, doGet(s, "/api/inbox").Body.Bytes()) + r := resp.Items[0].Routes[inbox.DestSpekRFC] + if r == nil || r.Status != inbox.StatusFiled || r.URL != "https://github.com/o/r/issues/1" || len(r.Children) != 1 || resp.Unprocessed != 0 { + t.Fatalf("spek route = %+v unprocessed=%d", r, resp.Unprocessed) + } + rec = doPost(s, "/api/inbox/route", map[string]string{"source": "plaud", "item_id": "rec-1", "destination": "dibs"}) + if rec.Code != http.StatusAccepted { + t.Fatalf("dibs route = %d: %s", rec.Code, rec.Body.String()) + } + resp = decodeInboxList(t, doGet(s, "/api/inbox").Body.Bytes()) + if d := resp.Items[0].Routes[inbox.DestDibs]; d == nil || d.Status != inbox.StatusFailed || d.Error != "dibs down" { + t.Fatalf("dibs route = %+v", d) + } + + rec = doPost(s, "/api/inbox/dismiss", map[string]any{"items": []map[string]string{{"source": "plaud", "item_id": "rec-1"}}}) + if rec.Code != http.StatusOK { + t.Fatalf("dismiss = %d: %s", rec.Code, rec.Body.String()) + } + resp = decodeInboxList(t, doGet(s, "/api/inbox").Body.Bytes()) + if !resp.Items[0].Dismissed { + t.Fatalf("item not dismissed: %+v", resp.Items[0]) + } +} + +func TestInboxSettingsPersistRules(t *testing.T) { + s, deps := apiServer(t) + deps.Inbox = newInboxTestService(t, config.InboxConfig{}) + if rec := doPost(s, "/api/inbox/settings", map[string]any{"rules": []map[string]string{{"destination": "spek_rfc"}}}); rec.Code != http.StatusBadRequest { + t.Fatalf("rule without filter = %d, want 400: %s", rec.Code, rec.Body.String()) + } + if rec := doPost(s, "/api/inbox/settings", map[string]any{"spek_repo": "not-a-repo"}); rec.Code != http.StatusBadRequest { + t.Fatalf("bad repo = %d, want 400", rec.Code) + } + rec := doPost(s, "/api/inbox/settings", map[string]any{ + "rules": []map[string]string{{"source": "plaud", "title_prefix": "RFC:", "destination": "spek_rfc", "repo": "o/r"}}, + "spek_repo": "o/r", + "dibs_url": "https://dibs.example/", + }) + if rec.Code != http.StatusOK { + t.Fatalf("settings = %d: %s", rec.Code, rec.Body.String()) + } + var st inbox.Settings + if err := json.Unmarshal(rec.Body.Bytes(), &st); err != nil { + t.Fatal(err) + } + if len(st.Rules) != 1 || st.Rules[0].TitlePrefix != "RFC:" || st.SpekRepo != "o/r" || st.DibsURL != "https://dibs.example" { + t.Fatalf("settings = %+v", st) + } + if deps.Config.Inbox.Spek.Repo != "o/r" || len(deps.Config.Inbox.Rules) != 1 || deps.Inbox.Config().Spek.Repo != "o/r" { + t.Fatalf("config not updated: %+v", deps.Config.Inbox) + } + if rec := doGet(s, "/api/inbox/settings"); rec.Code != http.StatusOK { + t.Fatalf("settings get = %d", rec.Code) + } +} + +// completerConn is an Authenticator that also accepts pasted completions. +type completerConn struct { + authConn + got connector.AuthCompletion +} + +func (c *completerConn) CompleteAuth(_ context.Context, comp connector.AuthCompletion) (connector.AuthState, error) { + c.got = comp + if comp.Tokens == "" && comp.RedirectURL == "" { + return c.state, errors.New("nothing pasted") + } + c.state = connector.AuthState{Connected: true, Account: "andy@example.com", CheckedAt: time.Now()} + return c.state, nil +} + +func TestKnowledgeConnectorAuthComplete(t *testing.T) { + s, deps := apiServer(t) + deps.KnowledgeConnectors = newTestConnectorSyncer(t, &authConn{}) + if rec := doPost(s, "/api/knowledge/connectors/wiki/auth/complete", map[string]string{"tokens": "{}"}); rec.Code != http.StatusBadRequest { + t.Fatalf("non-completer = %d, want 400: %s", rec.Code, rec.Body.String()) + } + conn := &completerConn{} + deps.KnowledgeConnectors = newTestConnectorSyncer(t, conn) + if rec := doPostNoOwner(s, "/api/knowledge/connectors/wiki/auth/complete", map[string]string{"tokens": "{}"}); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner = %d, want 403", rec.Code) + } + if rec := doPost(s, "/api/knowledge/connectors/wiki/auth/complete", map[string]string{}); rec.Code != http.StatusBadGateway { + t.Fatalf("empty completion = %d, want 502: %s", rec.Code, rec.Body.String()) + } + rec := doPost(s, "/api/knowledge/connectors/wiki/auth/complete", map[string]string{"redirect_url": "http://localhost:8199/auth/callback?code=x&state=y"}) + if rec.Code != http.StatusOK { + t.Fatalf("complete = %d: %s", rec.Code, rec.Body.String()) + } + if st := decodeAuthState(t, rec.Body.Bytes()); !st.Connected || conn.got.RedirectURL == "" { + t.Fatalf("state = %+v got = %+v", st, conn.got) + } +} diff --git a/src/pkg/dashboard/api_knowledge_connector_auth.go b/src/pkg/dashboard/api_knowledge_connector_auth.go new file mode 100644 index 0000000000..a5bd882be1 --- /dev/null +++ b/src/pkg/dashboard/api_knowledge_connector_auth.go @@ -0,0 +1,114 @@ +package dashboard + +import ( + "context" + "net/http" + "time" + + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// knowledgeConnectorLookup is implemented by the connector syncer so the +// dashboard can reach a connector's optional interfaces (Authenticator). +type knowledgeConnectorLookup interface { + Connector(name string) (connector.Connector, bool) +} + +// knowledgeAuthTimeout bounds one auth status/connect/disconnect request, +// which may spawn the connector's MCP server. +const knowledgeAuthTimeout = 90 * time.Second + +// Interactive connector sign-in (Plaud OAuth through its MCP server): status, +// connect (returns the authorization URL) and disconnect. Tokens stay in the +// vendor tool's own store and are never returned. +func (s *Server) registerKnowledgeConnectorAuthRoutes() { + s.mux.HandleFunc("GET /api/knowledge/connectors/{name}/auth", s.handleKnowledgeConnectorAuthStatus) + s.mux.HandleFunc("POST /api/knowledge/connectors/{name}/auth/connect", s.handleKnowledgeConnectorAuthConnect) + s.mux.HandleFunc("POST /api/knowledge/connectors/{name}/auth/disconnect", s.handleKnowledgeConnectorAuthDisconnect) +} + +// knowledgeAuthenticator resolves the named connector's Authenticator, +// writing the matching error response when it is unavailable. +func (s *Server) knowledgeAuthenticator(w http.ResponseWriter, name string) (connector.Authenticator, bool) { + rt := s.knowledgeConnectors() + if rt == nil { + jsonError(w, "no knowledge connectors are configured", http.StatusNotFound) + return nil, false + } + lookup, ok := rt.(knowledgeConnectorLookup) + if !ok { + jsonError(w, "connector sign-in is not available on this runtime", http.StatusServiceUnavailable) + return nil, false + } + c, ok := lookup.Connector(name) + if !ok { + jsonError(w, "unknown knowledge connector", http.StatusNotFound) + return nil, false + } + auth, ok := c.(connector.Authenticator) + if !ok { + jsonError(w, "this connector type does not use interactive sign-in", http.StatusBadRequest) + return nil, false + } + return auth, true +} + +func knowledgeAuthContext(r *http.Request) (context.Context, context.CancelFunc) { + return context.WithTimeout(context.WithoutCancel(r.Context()), knowledgeAuthTimeout) +} + +// handleKnowledgeConnectorAuthStatus returns the cached sign-in state, or a +// fresh one when ?refresh=1 (which may start the MCP server briefly). +func (s *Server) handleKnowledgeConnectorAuthStatus(w http.ResponseWriter, r *http.Request) { + auth, ok := s.knowledgeAuthenticator(w, r.PathValue("name")) + if !ok { + return + } + refresh := r.URL.Query().Get("refresh") == "1" + ctx, cancel := knowledgeAuthContext(r) + defer cancel() + jsonResponse(w, auth.AuthStatus(ctx, refresh)) +} + +// handleKnowledgeConnectorAuthConnect (owner) starts the sign-in flow and +// answers 202 with the state; auth_url is the page the owner must open. +func (s *Server) handleKnowledgeConnectorAuthConnect(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + name := r.PathValue("name") + auth, ok := s.knowledgeAuthenticator(w, name) + if !ok { + return + } + ctx, cancel := knowledgeAuthContext(r) + defer cancel() + st, err := auth.Connect(ctx) + if err != nil { + jsonError(w, "connect failed: "+err.Error(), http.StatusBadGateway) + return + } + s.auditFromRequest(r, "knowledge_connector_connect", auditDetail("connector", name), "") + jsonStatusResponse(w, http.StatusAccepted, st) +} + +// handleKnowledgeConnectorAuthDisconnect (owner) signs out upstream. +func (s *Server) handleKnowledgeConnectorAuthDisconnect(w http.ResponseWriter, r *http.Request) { + if !requireOwnerRole(w, r) { + return + } + name := r.PathValue("name") + auth, ok := s.knowledgeAuthenticator(w, name) + if !ok { + return + } + ctx, cancel := knowledgeAuthContext(r) + defer cancel() + st, err := auth.Disconnect(ctx) + if err != nil { + jsonError(w, "disconnect failed: "+err.Error(), http.StatusBadGateway) + return + } + s.auditFromRequest(r, "knowledge_connector_disconnect", auditDetail("connector", name), "") + jsonResponse(w, st) +} diff --git a/src/pkg/dashboard/api_knowledge_connector_auth_test.go b/src/pkg/dashboard/api_knowledge_connector_auth_test.go new file mode 100644 index 0000000000..7a7b7239f2 --- /dev/null +++ b/src/pkg/dashboard/api_knowledge_connector_auth_test.go @@ -0,0 +1,81 @@ +package dashboard + +import ( + "context" + "encoding/json" + "net/http" + "testing" + + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// authConn is a fake connector implementing connector.Authenticator. +type authConn struct { + blockingConn + state connector.AuthState +} + +func (c *authConn) AuthStatus(context.Context, bool) connector.AuthState { return c.state } +func (c *authConn) Connect(context.Context) (connector.AuthState, error) { + c.state = connector.AuthState{Pending: true, AuthURL: "https://auth.example/authorize"} + return c.state, nil +} +func (c *authConn) Disconnect(context.Context) (connector.AuthState, error) { + c.state = connector.AuthState{Message: "signed out"} + return c.state, nil +} + +func decodeAuthState(t *testing.T, body []byte) connector.AuthState { + t.Helper() + var st connector.AuthState + if err := json.Unmarshal(body, &st); err != nil { + t.Fatalf("decode: %v (%s)", err, body) + } + return st +} + +func TestKnowledgeConnectorAuthEndpoints(t *testing.T) { + s, deps := apiServer(t) + if rec := doGet(s, "/api/knowledge/connectors/wiki/auth"); rec.Code != http.StatusNotFound { + t.Fatalf("auth without syncer = %d, want 404", rec.Code) + } + + conn := &authConn{state: connector.AuthState{Connected: true, Account: "andy@example.com"}} + deps.KnowledgeConnectors = newTestConnectorSyncer(t, conn) + + rec := doGet(s, "/api/knowledge/connectors/wiki/auth") + if rec.Code != http.StatusOK { + t.Fatalf("auth status = %d: %s", rec.Code, rec.Body.String()) + } + if st := decodeAuthState(t, rec.Body.Bytes()); !st.Connected || st.Account != "andy@example.com" { + t.Fatalf("state = %+v", st) + } + if rec := doGet(s, "/api/knowledge/connectors/nope/auth"); rec.Code != http.StatusNotFound { + t.Fatalf("unknown connector = %d, want 404", rec.Code) + } + if rec := doPostNoOwner(s, "/api/knowledge/connectors/wiki/auth/connect", nil); rec.Code != http.StatusForbidden { + t.Fatalf("non-owner connect = %d, want 403", rec.Code) + } + rec = doPost(s, "/api/knowledge/connectors/wiki/auth/connect", nil) + if rec.Code != http.StatusAccepted { + t.Fatalf("connect = %d: %s", rec.Code, rec.Body.String()) + } + if st := decodeAuthState(t, rec.Body.Bytes()); !st.Pending || st.AuthURL != "https://auth.example/authorize" { + t.Fatalf("connect state = %+v", st) + } + rec = doPost(s, "/api/knowledge/connectors/wiki/auth/disconnect", nil) + if rec.Code != http.StatusOK { + t.Fatalf("disconnect = %d: %s", rec.Code, rec.Body.String()) + } + if st := decodeAuthState(t, rec.Body.Bytes()); st.Connected || st.Message != "signed out" { + t.Fatalf("disconnect state = %+v", st) + } +} + +func TestKnowledgeConnectorAuthNotInteractive(t *testing.T) { + s, deps := apiServer(t) + deps.KnowledgeConnectors = newTestConnectorSyncer(t, &blockingConn{}) + if rec := doGet(s, "/api/knowledge/connectors/wiki/auth"); rec.Code != http.StatusBadRequest { + t.Fatalf("non-authenticator connector = %d, want 400: %s", rec.Code, rec.Body.String()) + } +} diff --git a/src/pkg/dashboard/api_workflows_marketplace.go b/src/pkg/dashboard/api_workflows_marketplace.go index b7836e8b2a..f9ffcdea91 100644 --- a/src/pkg/dashboard/api_workflows_marketplace.go +++ b/src/pkg/dashboard/api_workflows_marketplace.go @@ -193,15 +193,29 @@ func (s *Server) importBundle(st *wfstore.Store, b *bundle.Bundle, by string) (w version = next } m.Spec.Meta.Version = version - if m.Spec.Meta.Provenance == "" { - m.Spec.Meta.Provenance = workflowmodel.Provenance(b.Tier) - } - if m.Spec.Meta.Provenance == workflowmodel.ProvenanceBuiltIn || string(m.Spec.Meta.Provenance) == bundle.TierLocalPublished { - m.Spec.Meta.Provenance = workflowmodel.ProvenanceLocal - } + m.Spec.Meta.Provenance = importedProvenance(m.Spec.Meta.Provenance, b.Tier) return st.Save(m, by, time.Now().UTC()) } +// importedProvenance decides what provenance an imported model is stored +// with. The envelope tier is the field the hub polices (curated/built-in +// need a hub admin); the model's own provenance is hash-covered and set by +// whoever authored the model, so a vouched tier claimed there is honoured +// only when the envelope asserts the same tier. built-in and +// local-published always land as local: this hive did not ship the model +// and has not published it. +func importedProvenance(model workflowmodel.Provenance, tier string) workflowmodel.Provenance { + p := model + vouched := p == workflowmodel.ProvenanceCurated || p == workflowmodel.ProvenanceBuiltIn + if p == "" || (vouched && string(p) != tier) { + p = workflowmodel.Provenance(tier) + } + if p == workflowmodel.ProvenanceBuiltIn || string(p) == bundle.TierLocalPublished || p == "" { + p = workflowmodel.ProvenanceLocal + } + return p +} + // hubRequest builds an authenticated spoke→hub request. func (s *Server) hubRequest(ctx context.Context, method, rawURL string, body []byte) (*http.Request, error) { var rd io.Reader diff --git a/src/pkg/dashboard/api_workflows_marketplace_test.go b/src/pkg/dashboard/api_workflows_marketplace_test.go index eed2d9897b..735b3c0205 100644 --- a/src/pkg/dashboard/api_workflows_marketplace_test.go +++ b/src/pkg/dashboard/api_workflows_marketplace_test.go @@ -458,3 +458,58 @@ func TestMarketplaceBaseFallsBackToHubURL(t *testing.T) { t.Fatal("server without deps should be enabled but have no base") } } + +func TestImportedProvenance(t *testing.T) { + cases := []struct { + model workflowmodel.Provenance + tier string + want workflowmodel.Provenance + }{ + // Empty model provenance takes the envelope tier. + {"", bundle.TierCurated, workflowmodel.ProvenanceCurated}, + {"", bundle.TierLocalPublished, workflowmodel.ProvenanceLocal}, + {"", bundle.TierBuiltIn, workflowmodel.ProvenanceLocal}, + {"", "", workflowmodel.ProvenanceLocal}, + // A vouched tier the envelope agrees with is kept (built-in still lands as local). + {workflowmodel.ProvenanceCurated, bundle.TierCurated, workflowmodel.ProvenanceCurated}, + {workflowmodel.ProvenanceBuiltIn, bundle.TierBuiltIn, workflowmodel.ProvenanceLocal}, + // A vouched tier the envelope does not assert is laundering: fall back to the tier. + {workflowmodel.ProvenanceCurated, bundle.TierLocalPublished, workflowmodel.ProvenanceLocal}, + {workflowmodel.ProvenanceCurated, bundle.TierIngestedUnreviewed, workflowmodel.ProvenanceIngestedUnreviewed}, + {workflowmodel.ProvenanceCurated, "", workflowmodel.ProvenanceLocal}, + {workflowmodel.ProvenanceBuiltIn, bundle.TierLocalPublished, workflowmodel.ProvenanceLocal}, + // Lower tiers in the model are honoured as-is. + {workflowmodel.ProvenanceIngestedUnreviewed, bundle.TierLocalPublished, workflowmodel.ProvenanceIngestedUnreviewed}, + {workflowmodel.ProvenanceLocal, bundle.TierCurated, workflowmodel.ProvenanceLocal}, + } + for _, tc := range cases { + if got := importedProvenance(tc.model, tc.tier); got != tc.want { + t.Errorf("importedProvenance(%q, %q) = %q, want %q", tc.model, tc.tier, got, tc.want) + } + } +} + +// A bundle whose envelope the hub normalised to local-published but whose +// hash-covered model still claims curated must not import as curated. +func TestWorkflowImportDoesNotLaunderCuratedProvenance(t *testing.T) { + s := marketplaceServer(t, "") + m := techdocsModel(t) + m.Spec.Meta.Provenance = workflowmodel.ProvenanceCurated + b, err := bundle.Export(m, "spoke", time.Now()) + if err != nil { + t.Fatal(err) + } + b.Tier = bundle.TierLocalPublished + raw, _ := bundle.Encode(b) + rec := wfReq(s, http.MethodPost, "/api/workflows/import", string(raw), true) + if rec.Code != http.StatusCreated { + t.Fatalf("import: %d %s", rec.Code, rec.Body.String()) + } + got, _, err := s.deps.WorkflowStore.Get("cncf-techdocs", "1") + if err != nil { + t.Fatal(err) + } + if got.Spec.Meta.Provenance != workflowmodel.ProvenanceLocal { + t.Fatalf("laundered provenance = %q, want local", got.Spec.Meta.Provenance) + } +} diff --git a/src/pkg/dashboard/benchmarks_cost.go b/src/pkg/dashboard/benchmarks_cost.go new file mode 100644 index 0000000000..650523332f --- /dev/null +++ b/src/pkg/dashboard/benchmarks_cost.go @@ -0,0 +1,80 @@ +package dashboard + +import ( + "github.com/hivecommons/hive/pkg/dashboard/collect" + "github.com/hivecommons/hive/pkg/evidence/benchmarks" + "github.com/hivecommons/hive/pkg/tokens" +) + +// benchmarksCostLookup attributes kick spend to a benchmark record from the +// launch-cost collector: USD from the persisted daily rollup (every day the +// store has seen, live days at their latest share) and token counts from the +// in-window snapshot, each launch split evenly across the refs it named — +// the same accounting as /api/launch-cost by_ref[].share_usd. Nil collector +// means nothing is attributed. +func benchmarksCostLookup(lc *collect.LaunchCostCollector) benchmarks.CostLookup { + if lc == nil { + return nil + } + return func(refs []benchmarks.CostRef) (benchmarks.Cost, bool) { + want := map[benchmarks.CostRef]bool{} + for _, r := range refs { + want[r] = true + } + var usd float64 + var launches int + for _, day := range lc.DailyRollup() { + for _, ref := range day.ByRef { + if want[benchmarks.CostRef{Ref: ref.Ref, Kind: ref.Kind}] { + usd += ref.ShareUSD + launches += ref.Launches + } + } + } + if usd <= 0 { + return benchmarks.Cost{Basis: benchmarks.CostBasisUnknown}, false + } + c := benchmarks.Cost{ + Basis: benchmarks.CostBasisTokensXPricing, + USDEst: &usd, + Launches: launches, + PriceTableDate: tokens.PriceTableDate(), + } + if snap, ok := lc.Snapshot(); ok { + addLaunchTokens(&c, snap.ByLaunch, want) + } + return c, true + } +} + +// addLaunchTokens adds the per-ref token share of every in-window launch +// that named one of the wanted refs. Tokens are informational: usd_est is +// already priced by the collector. +func addLaunchTokens(c *benchmarks.Cost, launches []collect.LaunchCostEntry, want map[benchmarks.CostRef]bool) { + var in, out, read, write int64 + seen := false + for _, l := range launches { + n := int64(len(l.Refs)) + if n == 0 { + continue + } + hit := 0 + for _, r := range l.Refs { + if want[benchmarks.CostRef{Ref: r.Ref, Kind: r.Kind}] { + hit++ + } + } + if hit == 0 { + continue + } + seen = true + k := int64(hit) + in += l.Input * k / n + out += l.Output * k / n + read += l.CacheRead * k / n + write += l.CacheCreate * k / n + } + if seen { + c.InputTokens, c.OutputTokens, c.CacheReadTokens, c.CacheWriteTokens = &in, &out, &read, &write + } +} diff --git a/src/pkg/dashboard/benchmarks_cost_test.go b/src/pkg/dashboard/benchmarks_cost_test.go new file mode 100644 index 0000000000..be62d6712f --- /dev/null +++ b/src/pkg/dashboard/benchmarks_cost_test.go @@ -0,0 +1,43 @@ +package dashboard + +import ( + "context" + "testing" + + "github.com/hivecommons/hive/pkg/dashboard/collect" + "github.com/hivecommons/hive/pkg/evidence/benchmarks" + "github.com/hivecommons/hive/pkg/tokens" +) + +func TestBenchmarksCostLookupFromLaunchCost(t *testing.T) { + if benchmarksCostLookup(nil) != nil { + t.Fatal("nil collector must disable attribution") + } + summary, launches, _ := launchFixture(t) + lc := collect.NewLaunchCostCollector(&fakeLaunchTokens{summary: summary, launches: launches}, nil) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + lc.Start(ctx) + + lookup := benchmarksCostLookup(lc) + // Reviewer record: PR ref as the swarm spells it. + c, ok := lookup([]benchmarks.CostRef{{Ref: "o/r#9", Kind: tokens.LaunchRefKindPR}, {Ref: "r#9", Kind: tokens.LaunchRefKindPR}}) + if !ok || c.Basis != benchmarks.CostBasisTokensXPricing || c.USDEst == nil || *c.USDEst <= 0 { + t.Fatalf("pr ref not attributed: %+v", c) + } + if c.Launches != 1 || c.PriceTableDate != tokens.PriceTableDate() { + t.Fatalf("launches/price table: %+v", c) + } + if c.InputTokens == nil || *c.InputTokens != 50 || c.OutputTokens == nil || *c.OutputTokens != 5 { + t.Fatalf("tokens from snapshot: in=%v out=%v", c.InputTokens, c.OutputTokens) + } + // Author record closing issue 1: governor ref, issue kind. + c, ok = lookup([]benchmarks.CostRef{{Ref: "o/r#1", Kind: tokens.LaunchRefKindIssue}}) + if !ok || *c.USDEst <= 0 || *c.InputTokens != 200 { + t.Fatalf("issue ref not attributed: %+v", c) + } + // Unknown ref: nothing landed. + if c, ok = lookup([]benchmarks.CostRef{{Ref: "o/r#99", Kind: tokens.LaunchRefKindIssue}}); ok || c.Basis != benchmarks.CostBasisUnknown { + t.Fatalf("unrelated ref must be unknown: %+v", c) + } +} diff --git a/src/pkg/dashboard/cli_models.go b/src/pkg/dashboard/cli_models.go index a5f4ba0a5a..17dfd916e7 100644 --- a/src/pkg/dashboard/cli_models.go +++ b/src/pkg/dashboard/cli_models.go @@ -17,6 +17,7 @@ import ( "strings" "sync" "time" + "unicode/utf8" "github.com/hivecommons/hive/crux" "github.com/hivecommons/hive/crux/claude" @@ -1625,12 +1626,30 @@ func copilotSDKHelperExecErr(ctx context.Context, err error, stderr string) erro errCopilotSDKHelperTimeout, copilotSDKProbeTimeout, filepath.Base(copilotSDKHelperPath)) } msg := strings.TrimSpace(stderr) + if isBinaryStderr(msg) { + return fmt.Errorf("sdk helper: %w (stderr was binary: the kernel refused to exec the Copilot CLI ELF and sh tried to run it as a script — ENOEXEC, see the copilot exec shim; not a model-catalog fault)", err) + } if len(msg) > copilotSDKStderrLogLimit { msg = msg[:copilotSDKStderrLogLimit] } return fmt.Errorf("sdk helper: %w (stderr: %s)", err, msg) } +// isBinaryStderr reports stderr that is not text: invalid UTF-8 or control +// bytes other than whitespace and ESC (ANSI colour), as when sh dumps an ELF +// it could not exec. +func isBinaryStderr(s string) bool { + if !utf8.ValidString(s) { + return true + } + for _, r := range s { + if r < 0x20 && r != '\n' && r != '\r' && r != '\t' && r != 0x1b { + return true + } + } + return false +} + // copilotSDKModel mirrors one entry of the helper's stdout JSON. The helper // also emits name/efforts/defaultEffort for future use; discovery only needs // the id and the policy gate today. diff --git a/src/pkg/dashboard/cli_models_last_good_test.go b/src/pkg/dashboard/cli_models_last_good_test.go index 394627059e..63c30a806f 100644 --- a/src/pkg/dashboard/cli_models_last_good_test.go +++ b/src/pkg/dashboard/cli_models_last_good_test.go @@ -11,6 +11,7 @@ import ( "sync" "testing" "time" + "unicode/utf8" ) // newLastGoodTestCache builds a cache with a pinned clock and an isolated @@ -61,6 +62,26 @@ func TestCopilotSDKHelperExecErr_OtherKeepsStderr(t *testing.T) { } } +// TestCopilotSDKHelperExecErr_BinaryStderrNamed: an ELF dumped as stderr (#17735) +// must not reach the self-report as garbage; it is named as an ENOEXEC. +func TestCopilotSDKHelperExecErr_BinaryStderrNamed(t *testing.T) { + err := copilotSDKHelperExecErr(context.Background(), errors.New("exit status 1"), "copilot: 1: \x7fELF\x02\x01\x00\xff\xfe\x06") + got := err.Error() + if !strings.Contains(got, "ENOEXEC") || strings.Contains(got, "ELF\x02") || !utf8.ValidString(got) { + t.Fatalf("binary stderr not summarised: %q", got) + } +} + +// TestCopilotSDKHelperExecErr_ANSIStderrKept: coloured stderr is text, not an +// ELF dump — it must be passed through rather than mislabelled as ENOEXEC. +func TestCopilotSDKHelperExecErr_ANSIStderrKept(t *testing.T) { + err := copilotSDKHelperExecErr(context.Background(), errors.New("exit status 1"), "\x1b[31merror:\x1b[0m auth required\n") + got := err.Error() + if strings.Contains(got, "ENOEXEC") || !strings.Contains(got, "auth required") { + t.Fatalf("ANSI stderr mislabelled: %q", got) + } +} + // TestQueryCLIModels_SDKFailureServesLastGoodCatalog is the #11865 flap: a // clean SDK probe, then one that dies. The served list must stay the CLI's // catalog — authoritative, labelled stale with the helper failure as the diff --git a/src/pkg/dashboard/dashboard_section_cards_test.go b/src/pkg/dashboard/dashboard_section_cards_test.go index 11d6d11f5d..7fa8a5d0ac 100644 --- a/src/pkg/dashboard/dashboard_section_cards_test.go +++ b/src/pkg/dashboard/dashboard_section_cards_test.go @@ -274,6 +274,7 @@ func TestDashboardSectionRenderersRefreshSharedShell(t *testing.T) { "function renderFAQ()": "faq-section", "function renderInception()": "inception-section", "function renderKnowledge()": "knowledge-section", + "function renderInbox()": "inbox-section", "function renderDebugSection()": "debug-section", "function renderLogs()": "logs-section", } diff --git a/src/pkg/dashboard/dashboard_section_reorder_9062_test.go b/src/pkg/dashboard/dashboard_section_reorder_9062_test.go index 640347c5c0..d3997b7ddc 100644 --- a/src/pkg/dashboard/dashboard_section_reorder_9062_test.go +++ b/src/pkg/dashboard/dashboard_section_reorder_9062_test.go @@ -17,6 +17,7 @@ var dashboardLayoutIDs9062 = []string{ "activity-monitor-section", "knowledge-section", "search-section", + "inbox-section", "review-queue-section", "contributors-section", "advisory-section", @@ -43,6 +44,7 @@ var dashboardDefaultNavOrder9062 = []string{ "activity-monitor-section", "knowledge-section", "search-section", + "inbox-section", "review-queue-section", "contributors-section", "advisory-section", @@ -57,7 +59,7 @@ var dashboardDefaultNavOrder9062 = []string{ var dashboardDefaultLayoutOrder9062 = append([]string{}, dashboardLayoutIDs9062...) -const dashboardDefaultLayoutTemplate9062 = `DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','runs-section','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','search-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}` +const dashboardDefaultLayoutTemplate9062 = `DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','runs-section','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','search-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}` func jsArray9062(ids []string) string { var b strings.Builder @@ -319,7 +321,7 @@ if (middle !== 'nous-section,logs-section,agents-section,agent-activity-section, if (new Set(got).size !== got.length) throw new Error('duplicate ids: '+JSON.stringify(got)); const moved = dashboardLayoutNormalize({v:1, main:['faq-section','cost-panel','overview-section']}).main; if (moved[0] !== 'faq-section') throw new Error('operator-moved FAQ not respected: '+JSON.stringify(moved)); -if (moved.length !== 23) throw new Error('missing sections not restored: '+moved.length); +if (moved.length !== 24) throw new Error('missing sections not restored: '+moved.length); ` cmd := exec.Command(node, "-e", script) if out, err := cmd.CombinedOutput(); err != nil { diff --git a/src/pkg/dashboard/dashboard_section_summaries_test.go b/src/pkg/dashboard/dashboard_section_summaries_test.go index 53f124738c..e1f1a7f317 100644 --- a/src/pkg/dashboard/dashboard_section_summaries_test.go +++ b/src/pkg/dashboard/dashboard_section_summaries_test.go @@ -39,7 +39,7 @@ func TestDashboardSectionCardsHaveCollapsedSummaries(t *testing.T) { "overview-section", "advisory-section", "hive-advice-section", "fleet-report-section", "acmm-reco-section", "lifecycle-section", "pr-throughput-section", "repos-section", "acmm-eval-section", "approvals-section", "audit-section", - "review-queue-section", "nous-section", "inception-section", "knowledge-section", + "review-queue-section", "nous-section", "inception-section", "knowledge-section", "inbox-section", "contributors-section", "debug-section", "logs-section", "agents-section", "faq-section", } for _, id := range refreshRequired { diff --git a/src/pkg/dashboard/deployment_upgrade.go b/src/pkg/dashboard/deployment_upgrade.go index 1c15e90f3b..61f21bbec3 100644 --- a/src/pkg/dashboard/deployment_upgrade.go +++ b/src/pkg/dashboard/deployment_upgrade.go @@ -18,6 +18,7 @@ import ( "strings" "time" + "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/fleetreport" ) @@ -356,7 +357,7 @@ func kubernetesSelfUpgradeRBACPrecheck() error { } client := &http.Client{ Timeout: selfUpgradePrecheckTimeout, - Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}}, + Transport: &http.Transport{TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, DialContext: egressmark.Dialer(0).DialContext}, } path := fmt.Sprintf("/apis/apps/v1/namespaces/%s/deployments/%s", namespace, selfUpgradeDeploymentName) if err := kubernetesSelfUpgradeRequest(client, http.MethodGet, path, nil, token); err != nil { diff --git a/src/pkg/dashboard/deps.go b/src/pkg/dashboard/deps.go index 094414e08e..cc6a9e68be 100644 --- a/src/pkg/dashboard/deps.go +++ b/src/pkg/dashboard/deps.go @@ -28,6 +28,7 @@ import ( "github.com/hivecommons/hive/pkg/governor" "github.com/hivecommons/hive/pkg/hivestate" "github.com/hivecommons/hive/pkg/hooks" + "github.com/hivecommons/hive/pkg/inbox" "github.com/hivecommons/hive/pkg/knowledge" "github.com/hivecommons/hive/pkg/mention" "github.com/hivecommons/hive/pkg/nous" @@ -133,6 +134,9 @@ type Dependencies struct { // KnowledgePublish is the running knowledge.publish mirror; nil when // publishing is not configured or failed to start. KnowledgePublish KnowledgePublishRuntime + // Inbox is the voice-source triage service; nil when no voice + // connector is configured. + Inbox *inbox.Service // RotationMgr is the provider-rotation headroom reporter (RFC #3958). Nil // when rotation is disabled; the headroom endpoint then reports // enabled=false. diff --git a/src/pkg/dashboard/hub_activity.go b/src/pkg/dashboard/hub_activity.go index a278242a58..0010590830 100644 --- a/src/pkg/dashboard/hub_activity.go +++ b/src/pkg/dashboard/hub_activity.go @@ -63,7 +63,7 @@ func (s *Server) HubActivitySnapshot(ctx context.Context) (hubactivity.Snapshot, if status == nil { return hubactivity.Snapshot{}, false } - body, _, _, _ := s.stageTrackerPayloadFrom(ctx, status, time.Now().UTC(), false) + body, _, _, _ := s.stageTrackerPayloadFrom(ctx, status, time.Now().UTC(), false, true) if body == nil { return hubactivity.Snapshot{}, false } diff --git a/src/pkg/dashboard/hub_activity_test.go b/src/pkg/dashboard/hub_activity_test.go index ea8f730c73..206ecc34a4 100644 --- a/src/pkg/dashboard/hub_activity_test.go +++ b/src/pkg/dashboard/hub_activity_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/hivecommons/hive/internal/testutil" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/hubactivity" "github.com/hivecommons/hive/pkg/stagelabels" @@ -110,3 +111,56 @@ func TestOnStageTrackerPublishFiresOnInvalidation(t *testing.T) { var nilServer *Server nilServer.OnStageTrackerPublish(func() {}) } + +// TestHubActivityItemsServesStaleBodyWithoutBuilding pins +// hivecommons/hive#17776: once the tracker has a body, an invalidated cache +// must not make the digest read wait on a rebuild. +func TestHubActivityItemsServesStaleBodyWithoutBuilding(t *testing.T) { + t.Setenv("HIVE_PUBLIC_REPOS", "") + now := time.Date(2026, 10, 31, 9, 0, 0, 0, time.UTC) + s := newTestServer() + s.stageStore = stagelabels.NewStore("", func() time.Time { return now }) + s.statusMu.Lock() + s.status = minimalPayload() + s.statusMu.Unlock() + + release := make(chan struct{}) + started := make(chan struct{}, 2) + s.stageTrackerBuildFn = func(_ *StatusPayload, _ *stagelabels.Store, at time.Time) StageTrackerResponse { + started <- struct{}{} + <-release + return StageTrackerResponse{GeneratedAt: at, Items: []StageTrackerItem{{Repo: "acme/widget", Number: 7, Kind: "pr", Modifiers: []string{}}}} + } + close(release) + if _, ok := s.HubActivitySnapshot(context.Background()); !ok { + t.Fatal("first read must build a body when none exists") + } + <-started + + release = make(chan struct{}) + s.invalidateStageTrackerCache() + type result struct { + snap hubactivity.Snapshot + ok bool + } + done := make(chan result, 1) + go func() { + snap, ok := s.HubActivitySnapshot(context.Background()) + done <- result{snap, ok} + }() + select { + case r := <-done: + if !r.ok || len(r.snap.Items) != 1 { + t.Fatalf("stale read = %+v, %v; want the previous body", r.snap, r.ok) + } + case <-time.After(2 * time.Second): + t.Fatal("digest read blocked on the rebuild instead of serving the stale body") + } + <-started + close(release) + testutil.Eventually(t, 2*time.Second, func() bool { + s.stageTrackerCacheMu.Lock() + defer s.stageTrackerCacheMu.Unlock() + return s.stageTrackerBuild == nil + }, "background rebuild never finished") +} diff --git a/src/pkg/dashboard/import_count_test.go b/src/pkg/dashboard/import_count_test.go index b752cb31ac..3278bdb113 100644 --- a/src/pkg/dashboard/import_count_test.go +++ b/src/pkg/dashboard/import_count_test.go @@ -75,8 +75,11 @@ func TestDashboardInternalImportCountRatchet(t *testing.T) { // shared with the scheduler's kick injection (hivecommons/hive#17244), // plus pkg/workflowmodel/store, the spoke-local library of workflow // models behind the /api/workflows routes and the tracker's per-repo - // column block (RFC hivecommons/hive#17620, phase 2). - const maxDashboardInternalImports = 69 + // column block (RFC hivecommons/hive#17620, phase 2), plus pkg/inbox, + // the source-agnostic voice-recording triage service (items keyed by + // source + item id, spek_rfc/dibs destinations) behind the /api/inbox + // routes and the Inbox dashboard section. + const maxDashboardInternalImports = 70 entries, err := os.ReadDir(".") if err != nil { diff --git a/src/pkg/dashboard/inbox_loader_static_test.go b/src/pkg/dashboard/inbox_loader_static_test.go new file mode 100644 index 0000000000..1e4e75185a --- /dev/null +++ b/src/pkg/dashboard/inbox_loader_static_test.go @@ -0,0 +1,36 @@ +package dashboard + +import ( + "os" + "strings" + "testing" +) + +// The Inbox loader must never leave the "Loading inbox…" placeholder up: a +// non-OK /api/inbox response renders an error state, and an OK response +// with zero items renders the empty-state text (#17918). +func TestInboxLoaderHandlesNonOKAndEmpty(t *testing.T) { + src, err := os.ReadFile("static-src/js/inbox.js") + if err != nil { + t.Fatal(err) + } + js := string(src) + start := strings.Index(js, "async function fetchInbox()") + if start < 0 { + t.Fatal("fetchInbox missing") + } + body := js[start : strings.Index(js[start:], "dashboardScheduleWork('inbox'")+start] + if strings.Contains(body, "if (!r.ok) return;") { + t.Fatal("fetchInbox bails out on a non-OK response, leaving the loading placeholder in place") + } + for _, want := range []string{"if (!r.ok) {", "_inboxCache = { items: []", "renderInbox();"} { + if !strings.Contains(body, want) { + t.Fatalf("fetchInbox non-OK branch missing %q:\n%s", want, body) + } + } + for _, want := range []string{"if (data.error) html +=", "Nothing yet. Connect Plaud or another voice source"} { + if !strings.Contains(js, want) { + t.Fatalf("renderInbox missing %q", want) + } + } +} diff --git a/src/pkg/dashboard/inbox_settings_section_test.go b/src/pkg/dashboard/inbox_settings_section_test.go new file mode 100644 index 0000000000..147803693d --- /dev/null +++ b/src/pkg/dashboard/inbox_settings_section_test.go @@ -0,0 +1,34 @@ +package dashboard + +import ( + "strings" + "testing" +) + +func TestSettingsInboxSectionWiring(t *testing.T) { + raw, err := staticFS.ReadFile("static/index.html") + if err != nil { + t.Fatal(err) + } + page := string(raw) + for _, want := range []string{ + "'Knowledge', 'Inbox', 'Project Observability'", + "case 'Inbox': return renderGovInbox();", + "function renderGovInbox", + "function loadInboxSettingsInline", + "loadInboxSettingsInline();", + "inbox-settings-inline", + "yamlHint('inbox.spek.repo')", + "yamlHint('inbox.dibs.url')", + "yamlHint('inbox.rules')", + "Default RFC repository", + "data-inbox-field=\"dibs-url\"", + "function inboxSettingsRoot", + "data-inbox-form=", + "key: 'inbox.spek.repo'", + } { + if !strings.Contains(page, want) { + t.Errorf("Settings → Inbox wiring missing %q", want) + } + } +} diff --git a/src/pkg/dashboard/job_terminal.go b/src/pkg/dashboard/job_terminal.go index 032f93eff8..43800908c9 100644 --- a/src/pkg/dashboard/job_terminal.go +++ b/src/pkg/dashboard/job_terminal.go @@ -66,8 +66,20 @@ import ( // So while a kick runs the pane shows what the sandbox did around it // (network and filesystem decisions, supervisor state) and then points the // operator at the transcript, which is where the kick's own output and -// exit code land once the launcher's Exec().Run returns. Only the job pod -// log shows the kick's stdout live. +// exit code land once the launcher's Exec().Run returns. The job pod log +// and the substrate stream below show the kick's stdout live. +// +// Substrate runtime (hivecommons/hive#17777). An agent with +// sandbox.runtime: substrate runs each kick in a Substrate actor, again with +// no tmux session, so it uses the same page, route and three states. The +// Control API has no ListActors and Substrate no logs RPC, so the live +// source is sandbox.SubstrateLogReader: the launcher, in this same process, +// records the actor (name, atespace, every state it passes through, the +// worker's SPIFFE id) and tees the kick's stdout and stderr from the +// bridge's streamed /run into it. Unlike OpenShell this is the kick's own +// output and exit code. When the launcher is done the actor is deleted or, +// with snapshot_on_idle, suspended; the pane names which, and between kicks +// shows how the last actor ended above the persisted transcript. const ( // Both live under terminalPathPrefix so the terminal handoff code, @@ -99,6 +111,7 @@ type jobPodLogReader interface { type jobTerminalDeps struct { reader jobPodLogReader openshell sandbox.OpenShellLogReader + substrate sandbox.SubstrateLogReader workspaceRoot string pollInterval time.Duration } @@ -139,6 +152,15 @@ func (s *Server) openshellLogReader(name string) sandbox.OpenShellLogReader { return &sandbox.OpenShellLauncher{Options: opts} } +// substrateLogReader is the substrate counterpart: the process-wide record +// the substrate launcher writes each kick into. +func (s *Server) substrateLogReader() sandbox.SubstrateLogReader { + if s.jobTerminal.substrate != nil { + return s.jobTerminal.substrate + } + return sandbox.DefaultSubstrateKicks +} + func (s *Server) jobTerminalWorkspaceRoot() string { if s.jobTerminal.workspaceRoot != "" { return s.jobTerminal.workspaceRoot @@ -163,11 +185,11 @@ func (s *Server) jobTerminalPollInterval() time.Duration { // runtime runs its kicks outside the hive pod, so it has no tmux session // and its terminal is this read-only viewer. func sandboxRuntimeWithoutTmux(rt string) bool { - return rt == config.SandboxRuntimeJob || rt == config.SandboxRuntimeOpenShell + return rt == config.SandboxRuntimeJob || rt == config.SandboxRuntimeOpenShell || rt == config.SandboxRuntimeSubstrate } // jobTerminalAgent resolves the path's agent and checks that its effective -// sandbox runtime has no tmux session (job or openshell), returning that +// sandbox runtime has no tmux session (job, openshell or substrate), returning that // runtime. The status code says which precondition failed: 404 for an // unknown agent, 409 for one that has a tmux terminal instead. func (s *Server) jobTerminalAgent(r *http.Request) (string, string, int, string) { @@ -185,7 +207,7 @@ func (s *Server) jobTerminalAgent(r *http.Request) (string, string, int, string) } rt := agentCfg.SandboxRuntime(cfg.AgentSandbox) if !sandboxRuntimeWithoutTmux(rt) { - return "", "", http.StatusConflict, fmt.Sprintf("agent %q runs sandbox kicks with runtime %q, not as a Kubernetes Job or OpenShell sandbox; there is no sandbox log to show", name, rt) + return "", "", http.StatusConflict, fmt.Sprintf("agent %q runs sandbox kicks with runtime %q, not as a Kubernetes Job, OpenShell sandbox or Substrate actor; there is no sandbox log to show", name, rt) } return name, rt, http.StatusOK, "" } @@ -291,6 +313,10 @@ func (s *Server) handleJobTerminalStream(w http.ResponseWriter, r *http.Request) s.streamOpenShellSandbox(ctx, out, name, s.openshellLogReader(name)) return } + if rt == config.SandboxRuntimeSubstrate { + s.streamSubstrateActor(ctx, out, name, s.substrateLogReader()) + return + } reader := s.jobPodLogReader() pod, found, err := reader.LatestPod(ctx, sandbox.LaunchName(name)) @@ -407,6 +433,89 @@ func (s *Server) streamOpenShellSandbox(ctx context.Context, out *jobTerminalWri } } +// streamSubstrateActor is the substrate-runtime counterpart: name the +// agent's newest actor and its state and SPIFFE id, follow the kick's +// output as the bridge streams it, scrubbed line by line, with each actor +// state transition as a status line, then report the kick's exit code and +// whether the actor was suspended or deleted. A finished kick's actor is +// still named, above the persisted transcript. +func (s *Server) streamSubstrateActor(ctx context.Context, out *jobTerminalWriter, name string, reader sandbox.SubstrateLogReader) { + info, found, err := reader.LatestActor(ctx, sandbox.LaunchName(name)) + if err != nil { + out.status("cannot read Substrate actors for agent %q: %v", name, err) + out.status("showing the last persisted transcript instead") + s.writeSandboxTranscript(out, name, substrateTerms) + return + } + if !found { + s.writeSandboxTranscript(out, name, substrateTerms) + return + } + out.status("substrate actor %s", describeSubstrateActor(info)) + if info.Finished() { + out.status("%s", substrateActorEnd(info)) + s.writeSandboxTranscript(out, name, substrateTerms) + return + } + out.status("%s", substrateLiveLogNote) + last := info + scrub := &kubejob.LineFilterWriter{Dst: out, Fn: scrubLogLine} + final, err := reader.StreamRun(ctx, info.Name, s.jobTerminalPollInterval(), scrub, func(next sandbox.SubstrateActorInfo) { + if next.State == last.State && next.SpiffeID == last.SpiffeID { + return + } + last = next + out.status("actor %s", describeSubstrateActor(next)) + }) + _ = scrub.Close() + if err != nil { + if ctx.Err() != nil { + return + } + out.status("output stream ended with an error: %v", err) + } + if final.Name == "" { + final.Name = info.Name + } + if final.Exited { + out.status("kick exited with code %d", final.ExitCode) + } + out.status("%s", substrateActorEnd(final)) +} + +// describeSubstrateActor is "<name> · <STATE> · <spiffe id>", leaving out +// what the launcher has not learned yet. +func describeSubstrateActor(i sandbox.SubstrateActorInfo) string { + parts := []string{i.Name, orUnknown(i.State)} + if i.Atespace != "" { + parts[0] = i.Atespace + "/" + i.Name + } + if i.SpiffeID != "" { + parts = append(parts, i.SpiffeID) + } + return strings.Join(parts, " · ") +} + +// substrateActorEnd says what became of the actor once the launcher was done +// with it, or that it has not finished. +func substrateActorEnd(i sandbox.SubstrateActorInfo) string { + switch { + case i.Suspended: + return fmt.Sprintf("actor %s was suspended with a snapshot (snapshot_on_idle): the kick has finished. %s", i.Name, substrateKickResultNote) + case i.Done: + return fmt.Sprintf("actor %s was deleted by the launcher: the kick has finished. %s", i.Name, substrateKickResultNote) + default: + return fmt.Sprintf("actor %s · %s (exit code not reported). %s", i.Name, orUnknown(i.State), substrateKickResultNote) + } +} + +const ( + // substrateLiveLogNote opens every live substrate stream. + substrateLiveLogNote = "this is the kick's stdout and stderr as the actor's bridge streams /run, scrubbed; actor state changes are shown as they happen" + // substrateKickResultNote closes it. + substrateKickResultNote = "the full output and exit code are in the persisted transcript; reopen this pane to see them" +) + const ( // openshellLiveLogNote opens every live OpenShell stream so an operator // never mistakes the sandbox's event log for the kick's output. @@ -440,11 +549,20 @@ var ( live: "OpenShell sandbox", where: "on the gateway", gone: "the launcher deletes a sandbox when its kick finishes", } + substrateTerms = sandboxTerms{ + title: "substrate actor log", + about: "the kick runs in a Substrate actor, so this is the kick's output as the actor's bridge streams it while it runs and the kick's transcript afterwards", + live: "Substrate actor", where: "started by this hive process", + gone: "the launcher deletes, or with snapshot_on_idle suspends, an actor when its kick finishes", + } ) func sandboxTermsFor(rt string) sandboxTerms { - if rt == config.SandboxRuntimeOpenShell { + switch rt { + case config.SandboxRuntimeOpenShell: return openshellTerms + case config.SandboxRuntimeSubstrate: + return substrateTerms } return jobTerms } diff --git a/src/pkg/dashboard/job_terminal_substrate_test.go b/src/pkg/dashboard/job_terminal_substrate_test.go new file mode 100644 index 0000000000..9db874f5f2 --- /dev/null +++ b/src/pkg/dashboard/job_terminal_substrate_test.go @@ -0,0 +1,250 @@ +package dashboard + +import ( + "context" + "errors" + "io" + "net/http" + "os" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/crux/sandbox" + "github.com/hivecommons/hive/pkg/config" +) + +// fakeSubstrateLogReader scripts sandbox.SubstrateLogReader: an optional +// recorded actor, the states it passes through, the output its bridge +// streams and the state it ends in. +type fakeSubstrateLogReader struct { + mu sync.Mutex + info sandbox.SubstrateActorInfo + found bool + latestErr error + states []sandbox.SubstrateActorInfo + lines []string + final sandbox.SubstrateActorInfo + streamErr error + launchNames []string + streamed []string + polls []time.Duration +} + +func (f *fakeSubstrateLogReader) LatestActor(_ context.Context, launchName string) (sandbox.SubstrateActorInfo, bool, error) { + f.mu.Lock() + defer f.mu.Unlock() + f.launchNames = append(f.launchNames, launchName) + return f.info, f.found, f.latestErr +} + +func (f *fakeSubstrateLogReader) StreamRun(_ context.Context, name string, poll time.Duration, w io.Writer, onState func(sandbox.SubstrateActorInfo)) (sandbox.SubstrateActorInfo, error) { + f.mu.Lock() + f.streamed = append(f.streamed, name) + f.polls = append(f.polls, poll) + states, lines, final, err := f.states, f.lines, f.final, f.streamErr + f.mu.Unlock() + for _, st := range states { + onState(st) + } + for _, l := range lines { + if _, werr := io.WriteString(w, l); werr != nil { + return final, werr + } + } + return final, err +} + +// substrateTerminalServer adds a substrate-runtime agent "actor" to the +// job-terminal fixture (which already has job "deps", podman "lint" and the +// tmux agent "scanner"). +func substrateTerminalServer(t *testing.T, reader *fakeSubstrateLogReader) (*Server, string) { + t.Helper() + s, root := jobTerminalServer(t, &fakeJobPodLogReader{}) + on := true + s.deps.Config.Agents["actor"] = config.AgentConfig{Backend: "claude", Enabled: true, Sandbox: &config.AgentSandboxOverride{Enabled: &on, Runtime: config.SandboxRuntimeSubstrate}} + s.jobTerminal.substrate = reader + return s, root +} + +const testSpiffeID = "spiffe://ate.local/ns/ate-workers/sa/worker" + +func TestJobTerminalGateAdmitsSubstrateRuntime(t *testing.T) { + if !sandboxRuntimeWithoutTmux(config.SandboxRuntimeSubstrate) { + t.Fatal("substrate agents have no tmux session; the viewer must admit them") + } + s, _ := substrateTerminalServer(t, &fakeSubstrateLogReader{}) + rec := doGet(s, "/terminal/job/actor") + if rec.Code != http.StatusOK { + t.Fatalf("substrate agent page: status = %d (%s)", rec.Code, rec.Body.String()) + } + body := rec.Body.String() + for _, want := range []string{"actor — substrate actor log", "Substrate actor", "'./actor/stream'"} { + if !strings.Contains(body, want) { + t.Errorf("substrate page missing %q", want) + } + } + if strings.Contains(body, "Kubernetes Job") || strings.Contains(body, "OpenShell") { + t.Fatal("substrate page must not describe the kick as a Job or OpenShell sandbox") + } + if rec := doGet(s, "/terminal/job/lint/stream"); rec.Code != http.StatusConflict || !strings.Contains(rec.Body.String(), "Substrate actor") { + t.Fatalf("podman agent stream: status = %d (%s)", rec.Code, rec.Body.String()) + } +} + +func TestSubstrateTerminalStreamsKickWithActorTransitionsScrubbed(t *testing.T) { + token := "ghp_" + strings.Repeat("Z", 30) + reader := &fakeSubstrateLogReader{ + info: sandbox.SubstrateActorInfo{Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateResuming}, + found: true, + states: []sandbox.SubstrateActorInfo{ + {Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateRunning, SpiffeID: testSpiffeID}, + // Only the exit code changed: no new status line. + {Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateRunning, SpiffeID: testSpiffeID, Exited: true, ExitCode: 4}, + {Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateSuspended, SpiffeID: testSpiffeID, Exited: true, ExitCode: 4, Suspended: true, Done: true}, + }, + lines: []string{"step 1\n", "auth " + token + " ok\n", "partial"}, + final: sandbox.SubstrateActorInfo{Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateSuspended, SpiffeID: testSpiffeID, Exited: true, ExitCode: 4, Suspended: true, Done: true}, + } + s, _ := substrateTerminalServer(t, reader) + rec := doGet(s, "/terminal/job/actor/stream") + if rec.Code != http.StatusOK { + t.Fatalf("status = %d", rec.Code) + } + body := rec.Body.String() + for _, want := range []string{ + "substrate actor hive/hive-kick-actor-1 · RESUMING", + "this is the kick's stdout and stderr as the actor's bridge streams /run", + "actor hive/hive-kick-actor-1 · RUNNING · " + testSpiffeID, + "actor hive/hive-kick-actor-1 · SUSPENDED · " + testSpiffeID, + "step 1\n", + "auth ghp_", + "partial", + // Unlike OpenShell, the bridge's exit code is the kick's own. + "kick exited with code 4", + "actor hive-kick-actor-1 was suspended with a snapshot (snapshot_on_idle): the kick has finished", + } { + if !strings.Contains(body, want) { + t.Errorf("missing %q in:\n%s", want, body) + } + } + if n := strings.Count(body, "· RUNNING ·"); n != 1 { + t.Errorf("RUNNING reported %d times; a state is reported once per transition:\n%s", n, body) + } + if strings.Contains(body, token) { + t.Fatalf("live kick output reached the browser unscrubbed:\n%s", body) + } + reader.mu.Lock() + defer reader.mu.Unlock() + if len(reader.launchNames) != 1 || reader.launchNames[0] != sandbox.LaunchName("actor") { + t.Fatalf("actor must be looked up by the executor's launch name, got %v", reader.launchNames) + } + if len(reader.streamed) != 1 || reader.streamed[0] != "hive-kick-actor-1" || reader.polls[0] != time.Millisecond { + t.Fatalf("streamed %v at %v", reader.streamed, reader.polls) + } +} + +func TestSubstrateTerminalStreamReportsErrorAndDeletedActor(t *testing.T) { + reader := &fakeSubstrateLogReader{ + info: sandbox.SubstrateActorInfo{Name: "a", State: sandbox.SubstrateStateRunning}, found: true, + final: sandbox.SubstrateActorInfo{Name: "a", State: sandbox.SubstrateStateDeleting, Done: true}, + streamErr: errors.New("router refused connect"), + } + s, _ := substrateTerminalServer(t, reader) + body := doGet(s, "/terminal/job/actor/stream").Body.String() + for _, want := range []string{"output stream ended with an error: router refused connect", "actor a was deleted by the launcher: the kick has finished. the full output and exit code are in the persisted transcript"} { + if !strings.Contains(body, want) { + t.Errorf("missing %q in:\n%s", want, body) + } + } + if strings.Contains(body, "kick exited with code") { + t.Fatalf("no exit code was reported; none must be shown:\n%s", body) + } + + reader = &fakeSubstrateLogReader{ + info: sandbox.SubstrateActorInfo{Name: "b", State: sandbox.SubstrateStateRunning}, found: true, + final: sandbox.SubstrateActorInfo{Name: "b", State: sandbox.SubstrateStateRunning}, + } + s, _ = substrateTerminalServer(t, reader) + if body := doGet(s, "/terminal/job/actor/stream").Body.String(); !strings.Contains(body, "actor b · RUNNING (exit code not reported)") { + t.Fatalf("unfinished stream:\n%s", body) + } +} + +// Between kicks the recorded actor is finished: the pane names it and how +// it ended (suspended with snapshot_on_idle), then shows the transcript. +func TestSubstrateTerminalFinishedActorShowsTranscript(t *testing.T) { + reader := &fakeSubstrateLogReader{ + info: sandbox.SubstrateActorInfo{Name: "hive-kick-actor-1", Atespace: "hive", State: sandbox.SubstrateStateSuspended, SpiffeID: testSpiffeID, Exited: true, Suspended: true, Done: true}, + found: true, + } + s, root := substrateTerminalServer(t, reader) + path := writeTranscript(t, root, "actor", 1, "stdout:\nfrom the last kick\n\nstderr:\n\n\n"+sandbox.TranscriptExitPrefix+"0\n") + body := doGet(s, "/terminal/job/actor/stream").Body.String() + for _, want := range []string{ + "substrate actor hive/hive-kick-actor-1 · SUSPENDED · " + testSpiffeID, + "actor hive-kick-actor-1 was suspended with a snapshot (snapshot_on_idle)", + path, "from the last kick", "last run exited with code 0", + } { + if !strings.Contains(body, want) { + t.Errorf("missing %q in:\n%s", want, body) + } + } + reader.mu.Lock() + streamed := reader.streamed + reader.mu.Unlock() + if len(streamed) != 0 { + t.Fatalf("finished actor: nothing must be streamed, got %v", streamed) + } +} + +func TestSubstrateTerminalFallsBackToTranscript(t *testing.T) { + reader := &fakeSubstrateLogReader{} + s, root := substrateTerminalServer(t, reader) + + body := doGet(s, "/terminal/job/actor/stream").Body.String() + if !strings.Contains(body, `no sandbox run yet for agent "actor": no Substrate actor started by this hive process`) || !strings.Contains(body, root) { + t.Fatalf("no-run state:\n%s", body) + } + + path := writeTranscript(t, root, "actor", 1, "stdout:\nfrom the last kick\n\nstderr:\n\n\n"+sandbox.TranscriptExitPrefix+"0\n") + body = doGet(s, "/terminal/job/actor/stream").Body.String() + for _, want := range []string{`no Substrate actor for agent "actor" right now`, path, "from the last kick", "last run exited with code 0"} { + if !strings.Contains(body, want) { + t.Errorf("missing %q in:\n%s", want, body) + } + } + + reader.latestErr = errors.New("registry unavailable") + body = doGet(s, "/terminal/job/actor/stream").Body.String() + for _, want := range []string{"cannot read Substrate actors", "registry unavailable", "from the last kick"} { + if !strings.Contains(body, want) { + t.Errorf("missing %q in:\n%s", want, body) + } + } +} + +func TestSubstrateTerminalDefaultReaderIsLauncherRecord(t *testing.T) { + s, _ := apiServer(t) + if r, ok := s.substrateLogReader().(*sandbox.SubstrateKicks); !ok || r != sandbox.DefaultSubstrateKicks { + t.Fatal("default reader must be the record the substrate launcher writes") + } +} + +func TestStaticTerminalLinkRoutesSubstrateRuntimeToViewer(t *testing.T) { + body, err := os.ReadFile("static/index.html") + if err != nil { + t.Fatal(err) + } + html := string(body) + for _, want := range []string{ + "const SUBSTRATE_SANDBOX_RUNTIME = 'substrate';", + "|| sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME;", + "if (a.sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME) {", + } { + if !strings.Contains(html, want) { + t.Fatalf("static dashboard substrate-terminal wiring missing %q", want) + } + } +} diff --git a/src/pkg/dashboard/knowledge_integrations_modal_test.go b/src/pkg/dashboard/knowledge_integrations_modal_test.go index 574593bc45..8dc4184b81 100644 --- a/src/pkg/dashboard/knowledge_integrations_modal_test.go +++ b/src/pkg/dashboard/knowledge_integrations_modal_test.go @@ -142,3 +142,89 @@ func TestKnowledgeIntegrationsModalContext7Card(t *testing.T) { } } } + +// TestKnowledgeIntegrationsModalPlaudCard guards the Plaud entry: the card +// exposes Connect/Disconnect/Check wired to the connector auth endpoints, +// the Knowledge pane links to it, and no browser-native dialog is used. +func TestKnowledgeIntegrationsModalPlaudCard(t *testing.T) { + b, err := staticFS.ReadFile("static/index.html") + if err != nil { + t.Fatalf("reading embedded static/index.html: %v", err) + } + html := string(b) + for _, want := range []string{ + `'plaud': { label: 'Plaud voice notes'`, + `'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest']`, + `if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name);`, + `data-action="kbPlaudPasteComplete"`, + `'/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/complete'`, + `if (type === 'plaud') html += kbPlaudAuthButtons(c.name);`, + `data-action="kbPlaudConnect"`, + `data-action="kbPlaudDisconnect"`, + `data-action="kbPlaudRefreshAuth"`, + `data-action="kbOpenPlaudIntegration"`, + `'/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/connect'`, + `'/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/disconnect'`, + `async function kbPlaudConnect(name)`, + `async function kbPlaudDisconnect(name)`, + `async function kbPlaudRefreshAuth(name)`, + `function kbOpenPlaudIntegration()`, + } { + if !strings.Contains(html, want) { + t.Errorf("index.html missing Plaud card snippet %q", want) + } + } + start := strings.Index(html, "const _kbPlaudAuth = {};") + end := strings.Index(html, "function kbOpenPlaudIntegration()") + if start < 0 || end < start { + t.Fatal("Plaud sign-in block not found") + } + block := html[start:end] + for _, native := range []string{"window.confirm(", "window.alert(", " confirm(", " alert(", "prompt("} { + if strings.Contains(block, native) { + t.Errorf("Plaud sign-in block uses a browser-native dialog %q", native) + } + } +} + +// TestKnowledgeIntegrationsModalWisprFlowCard guards the Wispr Flow entry: +// it is a plain local-file connector (no sign-in buttons), the card shows +// the configured path, the Knowledge pane links to it and documents the +// direct-dictation path, and no browser-native dialog is used. +func TestKnowledgeIntegrationsModalWisprFlowCard(t *testing.T) { + b, err := staticFS.ReadFile("static/index.html") + if err != nil { + t.Fatalf("reading embedded static/index.html: %v", err) + } + html := string(b) + for _, want := range []string{ + `'wisprflow': { label: 'Wispr Flow dictation'`, + `'wisprflow': [['path', 'flow.sqlite or export folder (empty = app default)'`, + `['group', 'Fact per dictation or day', 'dictation']`, + `if (type === 'wisprflow') html += kbWisprFlowSourceLine(c);`, + `data-kb-wisprflow-note`, + `data-action="kbOpenWisprFlowIntegration"`, + `function kbWisprFlowSourceLine(c)`, + `function kbOpenWisprFlowIntegration()`, + `Wispr Flow dictation</summary>`, + `+ New Fact</strong> and hold the Flow key`, + } { + if !strings.Contains(html, want) { + t.Errorf("index.html missing Wispr Flow card snippet %q", want) + } + } + if !strings.Contains(html, `'sharepoint', 'wisprflow']`) { + t.Error("wisprflow missing from KB_CONNECTOR_TYPE_ORDER") + } + start := strings.Index(html, "function kbWisprFlowSourceLine(c)") + end := strings.Index(html, "function kbOpenWisprFlowIntegration()") + if start < 0 || end < start { + t.Fatal("Wispr Flow block not found") + } + block := html[start:end] + for _, native := range []string{"window.confirm(", "window.alert(", " confirm(", " alert(", "prompt("} { + if strings.Contains(block, native) { + t.Errorf("Wispr Flow block uses a browser-native dialog %q", native) + } + } +} diff --git a/src/pkg/dashboard/repo_check_access_probe_test.go b/src/pkg/dashboard/repo_check_access_probe_test.go index 755d9047fc..3215153692 100644 --- a/src/pkg/dashboard/repo_check_access_probe_test.go +++ b/src/pkg/dashboard/repo_check_access_probe_test.go @@ -11,20 +11,14 @@ import ( "net/http/httptest" "os" "path/filepath" + "strconv" "strings" "testing" ghpkg "github.com/hivecommons/hive/pkg/github" ) -// These tests cover the App-present half of handleGovernorRepoCheckAccess — -// the DiscoverInstallationID probe the endpoint exists for. The App-less, -// cross-forge, and no-org guards are covered in repo_add_form_test.go; here a -// real *ghpkg.AppAuth (generated RSA key) talks to a stub GitHub API so both -// probe outcomes run deterministically and offline: -// - installed → 200 {ok:true, status:"ok", org} -// - not installed → 409 {ok:false, needsInstall:true, org, installUrl} and -// the pending-install banner mechanism is armed. +// App-present access checks use a real AppAuth and an offline GitHub stub. // newDiscoveryAppAuth builds an AppAuth whose JWT generation succeeds and whose // API calls hit the given handler. Discovery results are memoised process-wide @@ -59,8 +53,7 @@ func newDiscoveryAppAuth(t *testing.T, handler http.HandlerFunc) *ghpkg.AppAuth return auth } -// installedOrgHandler answers GET /orgs/{org}/installation affirmatively for -// exactly one org and 404s everything else (including the listing fallback). +// installedOrgHandler supplies an installation covering widget. func installedOrgHandler(org string, id int64) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if r.URL.Path == "/orgs/"+org+"/installation" { @@ -71,6 +64,16 @@ func installedOrgHandler(org string, id int64) http.HandlerFunc { }) return } + if r.URL.Path == "/app/installations/"+strconv.FormatInt(id, 10)+"/access_tokens" { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"token":"installation-token","expires_at":"2099-01-01T00:00:00Z"}`)) + return + } + if r.URL.Path == "/installation/repositories" { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(map[string]any{"repositories": []any{map[string]any{"full_name": org + "/widget"}}}) + return + } if r.URL.Path == "/app/installations" { w.Header().Set("Content-Type", "application/json") _, _ = w.Write([]byte(`[]`)) @@ -130,6 +133,16 @@ func TestGovernorRepoCheckAccess_FullURLResolvesOrg(t *testing.T) { } } +func TestGovernorRepoCheckAccess_BareRepoUsesConfiguredOrg(t *testing.T) { + srv := newFullServer(t) + srv.deps.Config.Project.Org = "bare-org" + srv.deps.GHAppAuth = newDiscoveryAppAuth(t, installedOrgHandler("bare-org", 123)) + w := checkAccessRequest(t, srv, `{"repo":"widget"}`) + if w.Code != http.StatusOK { + t.Fatalf("code = %d, want 200; body: %s", w.Code, w.Body.String()) + } +} + // App NOT installed on the org: 409 with needsInstall, the resolved org, the // per-forge install URL, and the pending-install banner mechanism armed so the // existing recheck plumbing drives the flow to completion. @@ -179,6 +192,112 @@ func TestGovernorRepoCheckAccess_NotInstalled409(t *testing.T) { } } +func TestGovernorRepoCheckAccess_RepoSelection(t *testing.T) { + for _, tc := range []struct { + name string + listStatus int + listBody string + wantStatus int + }{ + {"unselected", http.StatusOK, `{"repositories":[{"full_name":"selection-org/other"}]}`, http.StatusConflict}, + {"list failure", http.StatusInternalServerError, `{"message":"unavailable"}`, http.StatusServiceUnavailable}, + {"case insensitive", http.StatusOK, `{"repositories":[{"full_name":"SELECTION-ORG/WIDGET"}]}`, http.StatusOK}, + } { + t.Run(tc.name, func(t *testing.T) { + srv := newFullServer(t) + base := installedOrgHandler("selection-org", 4242) + srv.deps.GHAppAuth = newDiscoveryAppAuth(t, func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/installation/repositories" { + if r.Header.Get("Authorization") != "Bearer installation-token" { + t.Errorf("repository listing did not use installation token") + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tc.listStatus) + _, _ = w.Write([]byte(tc.listBody)) + return + } + base(w, r) + }) + w := checkAccessRequest(t, srv, `{"repo":"selection-org/widget"}`) + if w.Code != tc.wantStatus { + t.Fatalf("code = %d, want %d; body: %s", w.Code, tc.wantStatus, w.Body.String()) + } + if tc.wantStatus == http.StatusConflict { + var resp struct { + NeedsInstall bool `json:"needsInstall"` + InstallURL string `json:"installUrl"` + Error string `json:"error"` + } + if err := json.Unmarshal(w.Body.Bytes(), &resp); err != nil { + t.Fatal(err) + } + if !resp.NeedsInstall || !strings.Contains(resp.InstallURL, "/organizations/selection-org/settings/installations/4242") || !strings.Contains(resp.Error, "widget") { + t.Errorf("unexpected response: %+v", resp) + } + } + }) + } +} + +func TestGovernorRepoCheckAccess_DiscoveryFailure(t *testing.T) { + srv := newFullServer(t) + srv.deps.GHAppAuth = newDiscoveryAppAuth(t, func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "unavailable", http.StatusInternalServerError) + }) + w := checkAccessRequest(t, srv, `{"repo":"discovery-failure/widget"}`) + if w.Code != http.StatusServiceUnavailable || srv.IsPendingGitHubAppInstall() { + t.Fatalf("unexpected response %d %s; pending=%v", w.Code, w.Body.String(), srv.IsPendingGitHubAppInstall()) + } +} + +func TestGovernorRepoCheckAccess_TruncatedCoverageUsesRepositoryInstallation(t *testing.T) { + for _, tc := range []struct { + name string + status int + body string + wantStatus int + }{ + {"selected", http.StatusOK, `{"id":4242}`, http.StatusOK}, + {"not selected", http.StatusNotFound, `{"message":"Not Found"}`, http.StatusConflict}, + {"different installation", http.StatusOK, `{"id":2}`, http.StatusServiceUnavailable}, + {"missing installation id", http.StatusOK, `{}`, http.StatusServiceUnavailable}, + {"forbidden", http.StatusForbidden, `{"message":"rate limit"}`, http.StatusServiceUnavailable}, + {"rate limited", http.StatusTooManyRequests, `{"message":"rate limit"}`, http.StatusServiceUnavailable}, + {"server failure", http.StatusInternalServerError, `{"message":"unavailable"}`, http.StatusServiceUnavailable}, + } { + t.Run(tc.name, func(t *testing.T) { + srv := newFullServer(t) + base := installedOrgHandler("truncated-org", 4242) + probed := false + srv.deps.GHAppAuth = newDiscoveryAppAuth(t, func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/installation/repositories": + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Link", `<http://example.invalid/installation/repositories?page=2>; rel="next"`) + _, _ = w.Write([]byte(`{"repositories":[]}`)) + case "/repos/truncated-org/widget/installation": + probed = true + if auth := r.Header.Get("Authorization"); !strings.HasPrefix(auth, "Bearer ") || auth == "Bearer installation-token" { + t.Errorf("repository installation probe did not use App JWT: %q", auth) + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tc.status) + _, _ = w.Write([]byte(tc.body)) + default: + base(w, r) + } + }) + w := checkAccessRequest(t, srv, `{"repo":"truncated-org/widget"}`) + if !probed || w.Code != tc.wantStatus { + t.Fatalf("probed=%v code=%d, want %d; body: %s", probed, w.Code, tc.wantStatus, w.Body.String()) + } + if tc.wantStatus == http.StatusConflict && !strings.Contains(w.Body.String(), "/organizations/truncated-org/settings/installations/4242") { + t.Fatalf("missing installation settings link: %s", w.Body.String()) + } + }) + } +} + // Malformed body and blank repo are rejected before any probe runs. func TestGovernorRepoCheckAccess_BadInput(t *testing.T) { srv := newFullServer(t) diff --git a/src/pkg/dashboard/search_pane_ui_test.go b/src/pkg/dashboard/search_pane_ui_test.go index 3738882533..e2965dd39f 100644 --- a/src/pkg/dashboard/search_pane_ui_test.go +++ b/src/pkg/dashboard/search_pane_ui_test.go @@ -13,7 +13,7 @@ func TestSearchPaneRegisteredAsSection(t *testing.T) { for _, want := range []string{ `<a class="oc-nav-item" data-section="search-section" data-action="ocNavigate" data-arg0="search-section" hidden>`, `id="search-section" data-dashboard-section="search-section" style="display:none"`, - `'knowledge-section','search-section','review-queue-section'`, + `'knowledge-section','search-section','inbox-section','review-queue-section'`, `'search-section': { title: '🔍 Search', summary: 'search' },`, `'search-section': { heading: 'Search', help: `, `id="search-query" type="search"`, diff --git a/src/pkg/dashboard/server.go b/src/pkg/dashboard/server.go index 5537b9116b..985814591e 100644 --- a/src/pkg/dashboard/server.go +++ b/src/pkg/dashboard/server.go @@ -66,6 +66,9 @@ const agentSkipAfterFullBroadcastS = 5 * time.Second const maxSSEClients = 100 +// sseDropWarnInterval bounds how often dropped SSE events are logged at WARN. +const sseDropWarnInterval = 5 * time.Minute + const sessionCookieName = "hive_session" const sessionCookieMaxAge = 30 * 24 * 60 * 60 @@ -99,8 +102,10 @@ func isOwnerRole(role string) bool { // A WRONG proof header is ALWAYS rejected regardless of this setting. var proxyProofRequired = true -// staleDropStarvationThreshold is the number of consecutive stale-snapshot -// drops (no publish between) at which the drop is logged at Warn. +// staleDropStarvationThreshold bounds consecutive stale-snapshot drops (no +// publish between): the build that would be the Nth consecutive drop is +// published anyway, logged at Warn, so sustained mutation churn cannot +// starve consumers. The counter resets on every publish. const staleDropStarvationThreshold = 5 type Server struct { @@ -139,6 +144,8 @@ type Server struct { status *StatusPayload sseClients map[chan []byte]struct{} sseMu sync.Mutex + sseDropped int // events dropped since the last summary warning; guarded by sseMu + sseDropWarn time.Time // last summary warning; guarded by sseMu logger *slog.Logger mux *http.ServeMux deps *Dependencies @@ -1065,12 +1072,16 @@ type FrontendAgent struct { // SandboxRuntime is the effective sandbox.runtime for a sandboxed agent // ("podman" or "job"); the SPA routes "▶ terminal" to the Job pod log // viewer for "job" (hivecommons/hive#14798). Empty when not sandboxed. - SandboxRuntime string `json:"sandboxRuntime,omitempty"` - LastError string `json:"lastError,omitempty"` - StallNudges int `json:"stallNudges,omitempty"` - StallRecoveries int `json:"stallRecoveries,omitempty"` - ActionNudges int `json:"actionNudges,omitempty"` - TransientNudges int `json:"transientNudges,omitempty"` + SandboxRuntime string `json:"sandboxRuntime,omitempty"` + // SandboxWarnings are the boot/reload preflight findings for this agent's + // sandbox runtime (openshell or substrate), from the stored + // config.AgentSandboxGateReport; no preflight runs on the status path. + SandboxWarnings []string `json:"sandbox_warnings,omitempty"` + LastError string `json:"lastError,omitempty"` + StallNudges int `json:"stallNudges,omitempty"` + StallRecoveries int `json:"stallRecoveries,omitempty"` + ActionNudges int `json:"actionNudges,omitempty"` + TransientNudges int `json:"transientNudges,omitempty"` // SessionRssMb is the resident memory of the agent's CLI process tree // (MiB) and SessionAge how long the current CLI session has been up, // both sampled on the health tick (#14842). SessionRecycles counts @@ -3046,24 +3057,30 @@ func (s *Server) UpdateStatusIfFresh(status *StatusPayload, buildEpoch uint64) b s.statusMu.Lock() if buildEpoch < s.statusMutationEpoch { - // A mutation landed after this snapshot's build began: its data may - // predate the mutation. Drop it — the mutation's own refresh rebuild - // (which began after the epoch bump) publishes the fresh state. curEpoch := s.statusMutationEpoch - s.statusStaleDrops++ - drops := s.statusStaleDrops - s.statusConsecutiveStaleDrops++ - consecutive := s.statusConsecutiveStaleDrops - s.statusMu.Unlock() - // A single drop is normal optimistic-concurrency self-healing; only a - // run of drops with no publish in between is an anomaly. - level := slog.LevelInfo - if consecutive >= staleDropStarvationThreshold { - level = slog.LevelWarn + if s.statusConsecutiveStaleDrops+1 >= staleDropStarvationThreshold { + // Bounded staleness: under sustained mutation churn every build + // loses the race, so publish this one rather than starve + // consumers. The counter resets below with the publish. + drops := s.statusStaleDrops + consecutive := s.statusConsecutiveStaleDrops + s.logger.Warn("publishing possibly-stale status snapshot after consecutive stale drops", + "buildEpoch", buildEpoch, "mutationEpoch", curEpoch, "staleDrops", drops, "consecutiveDrops", consecutive) + } else { + // A mutation landed after this snapshot's build began: its data + // may predate the mutation. Drop it — the mutation's own refresh + // rebuild (which began after the epoch bump) publishes the fresh + // state. + s.statusStaleDrops++ + drops := s.statusStaleDrops + s.statusConsecutiveStaleDrops++ + consecutive := s.statusConsecutiveStaleDrops + s.statusMu.Unlock() + // A drop is a normal optimistic-concurrency race. + s.logger.Info("dropping stale status snapshot built before a mutation", + "buildEpoch", buildEpoch, "mutationEpoch", curEpoch, "staleDrops", drops, "consecutiveDrops", consecutive) + return false } - s.logger.Log(context.Background(), level, "dropping stale status snapshot built before a mutation", - "buildEpoch", buildEpoch, "mutationEpoch", curEpoch, "staleDrops", drops, "consecutiveDrops", consecutive) - return false } // Attach the advisory ACMM level-up recommendation (#5225) from the SAME @@ -4397,9 +4414,23 @@ func (s *Server) broadcastFrame(frame string) { select { case ch <- raw: default: - s.logger.Warn("SSE client too slow, dropping event") + s.sseDropped++ + s.logger.Debug("SSE client too slow, dropping event") } } + s.warnSSEDropsLocked(time.Now()) +} + +// warnSSEDropsLocked emits one summary warning per sseDropWarnInterval instead +// of one per dropped event, so a single stalled browser tab does not flood logs +// or trip repeated-error fleet reporting. Caller holds sseMu. +func (s *Server) warnSSEDropsLocked(now time.Time) { + if s.sseDropped == 0 || now.Sub(s.sseDropWarn) < sseDropWarnInterval { + return + } + s.logger.Warn("SSE clients too slow, dropped events", "dropped", s.sseDropped, "interval", sseDropWarnInterval.String()) + s.sseDropped = 0 + s.sseDropWarn = now } // initHistories lazily builds the three sparkline ring buffers. Called (once) diff --git a/src/pkg/dashboard/server_test.go b/src/pkg/dashboard/server_test.go index 3f0a0ad30e..dd49b1153a 100644 --- a/src/pkg/dashboard/server_test.go +++ b/src/pkg/dashboard/server_test.go @@ -2,6 +2,7 @@ package dashboard import ( "bufio" + "bytes" "encoding/json" "fmt" "log/slog" @@ -906,3 +907,23 @@ func TestNoAuth_AllEndpointsAccessible(t *testing.T) { t.Errorf("expected 200 when auth is not configured, got %d", resp.StatusCode) } } + +func TestBroadcast_SlowClientDropsWarnOncePerInterval(t *testing.T) { + var buf bytes.Buffer + s := newTestServer() + s.logger = slog.New(slog.NewTextHandler(&buf, &slog.HandlerOptions{Level: slog.LevelDebug})) + + full := make(chan []byte, 1) + full <- []byte("old-data") + s.sseMu.Lock() + s.sseClients[full] = struct{}{} + s.sseMu.Unlock() + + for i := 0; i < 20; i++ { + s.broadcastFrame("data: {}\n\n") + } + + if got := strings.Count(buf.String(), "level=WARN"); got != 1 { + t.Errorf("WARN lines: want 1, got %d\n%s", got, buf.String()) + } +} diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index fafa204332..b0e25e0296 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -527,6 +527,12 @@ review.perspective_models.<key>.backend # TODO #13243 review.perspective_models.<key>.model # TODO #13243 review.perspective_prompts.<key> # TODO #11854 review.perspectives[] # TODO #11854 +review.perspectives_source.owner # TODO #17975 +review.perspectives_source.path # TODO #17975 +review.perspectives_source.ref # TODO #17975 +review.perspectives_source.repo # TODO #17975 +review.perspectives_source.type # TODO #17975 +review.perspectives_source.url # TODO #17975 review.recommendations.labels[] # TODO #11854 review.recommendations.repos[] # TODO #11854 review.reviewer_agents[] # TODO #11854 @@ -750,3 +756,9 @@ project.workflow_bindings[].name # set through the per-repo workflow select on S project.workflow_bindings[].repo # see project.workflow_bindings[].name project.workflow_bindings[].version # see project.workflow_bindings[].name workflow.label_gc # operator safety knob for model-label garbage collection on rebind; yaml-only, no Settings control planned (RFC #17620) +inbox.sources[] # which voice connector types feed the Inbox; defaults to every configured plaud/wisprflow connector, yaml-only (Inbox RFC) +inbox.rules[].source # auto-routing rules are edited as whole rows in Inbox → Auto-routing via POST /api/inbox/settings, not leaf-path controls (Inbox RFC) +inbox.rules[].title_prefix # see inbox.rules[].source +inbox.rules[].tag # see inbox.rules[].source +inbox.rules[].destination # see inbox.rules[].source +inbox.rules[].repo # see inbox.rules[].source diff --git a/src/pkg/dashboard/stage_tracker_build.go b/src/pkg/dashboard/stage_tracker_build.go index 9887476949..9cbc900329 100644 --- a/src/pkg/dashboard/stage_tracker_build.go +++ b/src/pkg/dashboard/stage_tracker_build.go @@ -939,13 +939,16 @@ func (s *Server) stageTrackerPayload(status *StatusPayload, now time.Time) (body // came from (stageTrackerSource*Name; "" with a nil body when the build // failed or ctx ended while waiting on an in-flight build). func (s *Server) stageTrackerPayloadSource(ctx context.Context, status *StatusPayload, now time.Time) (body []byte, etag string, generatedAt time.Time, source string) { - return s.stageTrackerPayloadFrom(ctx, status, now, true) + return s.stageTrackerPayloadFrom(ctx, status, now, true, false) } // stageTrackerPayloadFrom is stageTrackerPayloadSource; markServed false // reads the tracker without counting as a client that asked for it, so an -// in-process reader (the hub activity digest) adds no SSE nudges. -func (s *Server) stageTrackerPayloadFrom(ctx context.Context, status *StatusPayload, now time.Time, markServed bool) (body []byte, etag string, generatedAt time.Time, source string) { +// in-process reader (the hub activity digest) adds no SSE nudges. preferStale +// serves a stale body at once when no build is in flight and rebuilds in the +// background, so a latency-bound reader is never the synchronous builder +// (hivecommons/hive#17776); with no body at all it still builds in place. +func (s *Server) stageTrackerPayloadFrom(ctx context.Context, status *StatusPayload, now time.Time, markServed, preferStale bool) (body []byte, etag string, generatedAt time.Time, source string) { ttl := s.stageTrackerCacheTTL() s.stageTrackerCacheMu.Lock() if markServed { @@ -979,6 +982,20 @@ func (s *Server) stageTrackerPayloadFrom(ctx context.Context, status *StatusPayl gen := s.stageTrackerGen s.stageTrackerCacheMu.Unlock() + if preferStale && len(c.body) > 0 { + go s.finishStageTrackerBuild(inflight, status, now, ttl, gen) + return c.body, c.etag, c.generatedAt, stageTrackerSourceStaleName + } + built, ok := s.finishStageTrackerBuild(inflight, status, now, ttl, gen) + if !ok { + return nil, "", now, "" + } + return built.body, built.etag, built.generatedAt, stageTrackerSourceFreshName +} + +// finishStageTrackerBuild runs the claimed in-flight build, stores the +// result and releases waiters. +func (s *Server) finishStageTrackerBuild(inflight *stageTrackerBuild, status *StatusPayload, now time.Time, ttl time.Duration, gen uint64) (stageTrackerCache, bool) { built, ok := s.encodeStageTracker(status, now, ttl) s.stageTrackerCacheMu.Lock() @@ -991,10 +1008,7 @@ func (s *Server) stageTrackerPayloadFrom(ctx context.Context, status *StatusPayl s.stageTrackerBuild = nil close(inflight.done) s.stageTrackerCacheMu.Unlock() - if !ok { - return nil, "", now, "" - } - return built.body, built.etag, built.generatedAt, stageTrackerSourceFreshName + return built, ok } // encodeStageTracker builds and encodes the tracker for status. It holds no diff --git a/src/pkg/dashboard/stage_tracker_human_gate_test.go b/src/pkg/dashboard/stage_tracker_human_gate_test.go index 53cec8f3a2..e21dc8e0a6 100644 --- a/src/pkg/dashboard/stage_tracker_human_gate_test.go +++ b/src/pkg/dashboard/stage_tracker_human_gate_test.go @@ -202,6 +202,13 @@ assert.deepStrictEqual(ids(['hold'], { kind: 'issue', stage: 10 }), ['Release ho assert.deepStrictEqual(stageTrackerHumanGateReasons(Object.assign({}, row140, { sentinel_notice: true })), []); assert.deepStrictEqual(stageTrackerHumanGateReasons(Object.assign({}, row17480, { modifiers: ['needs-decision', 'draft'] })), []); assert.deepStrictEqual(stageTrackerHumanGateReasons({ kind: 'pr', stage: 110, modifiers: ['merged', 'hold'] }), []); +// hivecommons/hive#17789: a 071 row flagged needs_human on a DCO sign-off +// ask is the author's move, not a decision; another 071 reason still is. +const row17780 = { repo: 'hivecommons/hive', number: 17780, kind: 'pr', stage: 71, head: '0e5ccbe', author: 'clubanderson', author_kind: 'human', needs_human: true, reason: 'hive-signoff-attribution' }; +assert.deepStrictEqual(stageTrackerHumanGateReasons(row17780), []); +assert.deepStrictEqual(stageTrackerHumanGateReasons(Object.assign({}, row17780, { reason: 'dco-missing' })), []); +assert.deepStrictEqual(stageTrackerHumanGateReasons(Object.assign({}, row17780, { reason: 'label:needs-rebase' })).map(r => r.key), ['decision']); +assert.deepStrictEqual(stageTrackerHumanGateReasons(Object.assign({}, row17780, { modifiers: ['needs-decision'] })).map(r => r.key), ['decision'], 'an explicit needs-decision still gates'); assert.ok(!stageTrackerRowHtml({ repo: 'acme/console', number: 1, kind: 'pr', stage: 50 }, stages, ctx).includes('ham-human-gate'), 'a clean row has no gate'); // (d) secondary rows #17463 / #17476 of the shared PR #17480 (primary row diff --git a/src/pkg/dashboard/stage_tracker_rework.go b/src/pkg/dashboard/stage_tracker_rework.go index fcd7511ea4..4dbb341738 100644 --- a/src/pkg/dashboard/stage_tracker_rework.go +++ b/src/pkg/dashboard/stage_tracker_rework.go @@ -112,9 +112,24 @@ func stageTrackerReworkOwedBy(it StageTrackerItem) string { if strings.Contains(strings.ToLower(it.Reason), "conflict") && it.Rebase == nil { return reworkOwedByAuthor } + // A DCO ask parks the head on the author's sign-off; no re-review is + // queued for it (hivecommons/hive#17789). + if stageTrackerAuthorDCOReason(it.Reason) { + return reworkOwedByAuthor + } return reworkOwedByHiveReviewer } +// stageTrackerAuthorDCOReason reports a lane reason that asks the author to +// fix their commits' sign-off. +func stageTrackerAuthorDCOReason(reason string) bool { + switch strings.TrimSpace(reason) { + case stagelabels.LaneReasonDCOMissing, stagelabels.LaneReasonHiveSignoffAttribution: + return true + } + return false +} + // StageTrackerFixWork is the hub's fix-queue standing for one PR and one // contributor (ContributeWSHub.FixWorkFor). type StageTrackerFixWork struct { diff --git a/src/pkg/dashboard/stage_tracker_rework_test.go b/src/pkg/dashboard/stage_tracker_rework_test.go index ce47bbd4cb..f6e85b1e1d 100644 --- a/src/pkg/dashboard/stage_tracker_rework_test.go +++ b/src/pkg/dashboard/stage_tracker_rework_test.go @@ -247,6 +247,10 @@ assert.strictEqual(stageTrackerPresenceChipText(working), 'waiting on @Danathar // A pushed head with nothing against it waits on the hive's reviewer (hivecommons/hive#16615). assert.strictEqual(stageTrackerPresenceChipText(pushed), 'waiting on 🐝 hive reviewer · re-review of bbb'); assert.strictEqual(stageTrackerPresenceChipText(w(Object.assign({}, pushed.rework, { owed_by: 'author' }))), 'waiting on @Danathar · re-review pending', 'the server says the author still owes it'); +// hivecommons/hive#17789: an older payload without owed_by reads a DCO ask as the author's move too. +assert.strictEqual(stageTrackerReReviewOwedByHive(Object.assign({}, pushed, { reason: 'hive-signoff-attribution' })), false); +assert.strictEqual(stageTrackerReReviewOwedByHive(Object.assign({}, pushed, { reason: 'dco-missing' })), false); +assert.strictEqual(stageTrackerPresenceChipText(Object.assign({}, pushed, { reason: 'hive-signoff-attribution', rework: Object.assign({}, pushed.rework, { owed_by: 'author' }) })), 'waiting on @Danathar · sign-off owed', 'no re-review is queued for a DCO ask'); assert.match(stageTrackerReworkTip(requested, NOW), /^rework requested 1 hour ago \(\S+ .*\) · no response from @Danathar yet$/); assert.match(stageTrackerReworkTip(queued, NOW), /^rework queued for @Danathar's clanker · 2 ahead · queued 10 min ago \(/); assert.match(stageTrackerReworkTip(acked, NOW), /^@Danathar acknowledged · activity on the PR 20 min ago \(/); @@ -305,6 +309,10 @@ func TestStageTrackerReworkOwedBy16615(t *testing.T) { "red CI on the pushed head": func(it *StageTrackerItem) { it.CIActivity.State = "failing" }, "findings on the pushed head": func(it *StageTrackerItem) { it.ReviewActivity.State = "changes_requested" }, "conflict nobody in the hive rebases": func(it *StageTrackerItem) { it.Reason = "conflicting" }, + // hivecommons/hive#17789: a DCO ask is the author's move; no + // re-review of the pushed head is queued for it. + "hive-signoff-attribution": func(it *StageTrackerItem) { it.Reason = "hive-signoff-attribution" }, + "dco-missing": func(it *StageTrackerItem) { it.Reason = "dco-missing" }, } { it := base() mutate(&it) diff --git a/src/pkg/dashboard/static-src/css/inbox.css b/src/pkg/dashboard/static-src/css/inbox.css new file mode 100644 index 0000000000..c81646cfae --- /dev/null +++ b/src/pkg/dashboard/static-src/css/inbox.css @@ -0,0 +1,17 @@ + /* Inbox (voice recordings / dictations triage) */ + .inbox-toolbar { flex-wrap: wrap; gap: var(--sp-3); margin-bottom: var(--sp-4); } + .inbox-sources { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); } + .inbox-toggle { display: inline-flex; align-items: center; gap: var(--sp-2); } + .inbox-list { max-height: none; } + .inbox-item { cursor: default; } + .inbox-check { display: flex; align-items: flex-start; padding-top: var(--sp-1); } + .inbox-icon { flex-shrink: 0; font-size: var(--fs-lg); line-height: 1; padding-top: var(--sp-1); } + .inbox-pills { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); margin-top: var(--sp-2); } + .inbox-link { font-size: var(--fs-xs); } + .inbox-field { display: block; margin-top: var(--sp-4); } + .inbox-settings-grid { display: grid; grid-template-columns: 1fr 1fr; gap: var(--sp-4); } + .inbox-rule { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1.4fr) minmax(0, 1fr) minmax(0, 1.2fr) minmax(0, 1.4fr) auto; gap: var(--sp-2); align-items: center; margin-bottom: var(--sp-2); } + .kb-plaud-paste textarea { width: 100%; margin-top: var(--sp-2); font-family: var(--font-mono); font-size: var(--fs-xs); } + @media (max-width: 720px) { + .inbox-settings-grid, .inbox-rule { grid-template-columns: 1fr; } + } diff --git a/src/pkg/dashboard/static-src/html/inbox.html b/src/pkg/dashboard/static-src/html/inbox.html new file mode 100644 index 0000000000..8b462ea673 --- /dev/null +++ b/src/pkg/dashboard/static-src/html/inbox.html @@ -0,0 +1,6 @@ + <div class="mt-6 mb-8" id="inbox-section" data-dashboard-section="inbox-section"> + <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" data-arg0="inbox-section"><button type="button" class="dashboard-grip" data-dashboard-grip data-stop="1" aria-label="Move Inbox section" aria-pressed="false">⠿</button><span class="section-chevron">▼</span> 📥 Inbox</h2> + <div class="section-body"> + <div id="inbox-panel" class="inbox-panel"></div> + </div> + </div> diff --git a/src/pkg/dashboard/static-src/html/sidebar.html b/src/pkg/dashboard/static-src/html/sidebar.html index d83f39b0a3..452c6725d3 100644 --- a/src/pkg/dashboard/static-src/html/sidebar.html +++ b/src/pkg/dashboard/static-src/html/sidebar.html @@ -44,6 +44,7 @@ <a class="oc-nav-item" data-section="inception-section" data-action="ocNavigate" data-arg0="inception-section"><span class="oc-nav-emoji">💡</span><span class="oc-nav-text">Inception (spektacular)</span></a> <a class="oc-nav-item" data-section="knowledge-section" data-action="ocNavigate" data-arg0="knowledge-section"><span class="oc-nav-emoji">📚</span><span class="oc-nav-text">Knowledge</span><span id="knowledge-sidebar-badge" class="badge-count oc-nav-count" data-zero>0</span></a> <a class="oc-nav-item" data-section="search-section" data-action="ocNavigate" data-arg0="search-section" hidden><span class="oc-nav-emoji">🔍</span><span class="oc-nav-text">Search</span></a> + <a class="oc-nav-item" data-section="inbox-section" data-action="ocNavigate" data-arg0="inbox-section"><span class="oc-nav-emoji">📥</span><span class="oc-nav-text">Inbox</span><span id="inbox-sidebar-badge" class="badge-count oc-nav-count" data-zero hidden>0</span></a> <a class="oc-nav-item" data-section="nous-section" data-feature="experiment_lab" hidden data-action="ocNavigate" data-arg0="nous-section"><span class="oc-nav-emoji">🧪</span><span class="oc-nav-text">Experiment Lab (nous)</span></a> </div> <div class="oc-nav-group"> diff --git a/src/pkg/dashboard/static-src/js/agent-cards.js b/src/pkg/dashboard/static-src/js/agent-cards.js index 0ba31f9f7e..b72e7734c4 100644 --- a/src/pkg/dashboard/static-src/js/agent-cards.js +++ b/src/pkg/dashboard/static-src/js/agent-cards.js @@ -99,11 +99,15 @@ by the Go server under the same /terminal prefix so openTerminal()'s handoff code and assertion cookie apply unchanged. openshell agents (#17645) run each kick in a gateway-owned sandbox and use the same - viewer, which streams the sandbox's log instead. */ + viewer, which streams the sandbox's log instead; substrate agents + (#17777) run each kick in a Substrate actor and the viewer streams the + kick's output from the actor's bridge. */ const JOB_SANDBOX_RUNTIME = 'job'; const OPENSHELL_SANDBOX_RUNTIME = 'openshell'; + const SUBSTRATE_SANDBOX_RUNTIME = 'substrate'; function sandboxRuntimeWithoutTmux(sandboxRuntime) { - return sandboxRuntime === JOB_SANDBOX_RUNTIME || sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME; + return sandboxRuntime === JOB_SANDBOX_RUNTIME || sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME + || sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME; } function agentUsesJobTerminal(a) { return !!a && sandboxRuntimeWithoutTmux(a.sandboxRuntime); @@ -119,6 +123,9 @@ } function terminalLinkTitle(a, tmuxTitle) { if (!agentUsesJobTerminal(a)) return tmuxTitle; + if (a.sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME) { + return 'Stream this agent\'s kick output and Substrate actor state (read-only: it runs in a Substrate actor, not a tmux session)'; + } return a.sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME ? 'Stream the log of this agent\'s OpenShell sandbox (read-only: it runs in a gateway-owned sandbox, not a tmux session)' : 'Stream the log of this agent\'s sandbox Job pod (read-only: it runs as a Kubernetes Job, not a tmux session)'; @@ -859,6 +866,12 @@ tmux session or in-pod process to be "down". Their health comes from the server-side sandboxRun verdict (last run + cadence), never from state !== 'running'. */ + /* Preflight findings for this agent's sandbox runtime (openshell or + substrate), computed at boot/reload and carried as sandbox_warnings. */ + function agentSandboxWarningsHtml(a) { + const ws = a && Array.isArray(a.sandbox_warnings) ? a.sandbox_warnings : []; + return ws.map(w => `<div class="agent-error" title="${esc(w)}">⚠ ${esc(w)}</div>`).join(''); + } function agentSandboxRun(a) { return (a && a.sandboxed && a.sandboxRun && a.sandboxRun.health) ? a.sandboxRun : null; } @@ -1098,6 +1111,7 @@ const prefix = retrying ? 'retrying: ' : ''; return `<div class="agent-error" title="${esc(a.lastError)}">⚠ ${esc(prefix)}${esc(a.lastError)}</div>`; })()} + ${agentSandboxWarningsHtml(a)} <div class="agent-card-details" data-agent-resize-y data-agent="${esc(a.name)}"${layoutHeightAttr}> <div class="agent-field"><span class="label" title="${isInferenceBackendName(a.cli) ? 'Inference backend (self-hosted model server)' : 'The CLI backend running this agent (e.g. claude, copilot). Pin to prevent governor from switching it.'}">${isInferenceBackendName(a.cli) ? 'method' : 'cli'}</span><span class="value">${cliChip(a.cli, a.pinnedBoth || a.pinnedCli, a.name)}${(a.pinnedBoth || a.pinnedCli) ? '' : ` <button class="hv-btn pin-toggle btn-icon btn-sm" data-action="togglePin" data-agent="${esc(a.name)}" data-pin-type="cli" data-unpin="false" title="Pin CLI — lock current backend">📌</button>`}</span></div> <div class="agent-field"><span class="label" title="The LLM model powering this agent. Governor may change it based on budget or mode. Pin to lock.">model</span><span class="value">${modelChip(a.model, a.govReason, a.pinnedBoth || a.pinnedModel, a.name)}${(a.pinnedBoth || a.pinnedModel) ? '' : ` <button class="hv-btn pin-toggle btn-icon btn-sm" data-action="togglePin" data-agent="${esc(a.name)}" data-pin-type="model" data-unpin="false" title="Pin model — lock current model">📌</button>`}</span></div> diff --git a/src/pkg/dashboard/static-src/js/config-agent.js b/src/pkg/dashboard/static-src/js/config-agent.js index 614365ac56..61a329c7fe 100644 --- a/src/pkg/dashboard/static-src/js/config-agent.js +++ b/src/pkg/dashboard/static-src/js/config-agent.js @@ -185,6 +185,9 @@ <div class="config-toggle-switch ${g.sandboxEnabled ? 'on' : ''}" id="cfg-agent-sandbox" data-action="toggleConfigSwitch" data-section="general" data-key="sandboxEnabled"></div> <span class="config-toggle-label">Run in sandbox ${settingHelpMark('sandbox-isolation', '', 'Run in sandbox', `Opt this agent into the credential-free sandbox runner. Requires the hive-wide sandbox gate in Governor → Security to be enabled.`)}${g.sandboxEffective ? ' ' + statusBadgeHTML('ready', 'sandboxed', '', 'done') : ''}</span> </div> + <details class="settings-subsection-details"><summary class="settings-subsection-summary">Substrate sandbox overrides</summary> + ${substrateSandboxFields('general', g.sandboxSubstrate)} + </details> <div class="config-toggle"> <div class="config-toggle-switch ${g.clearOnKick !== false ? 'on' : ''}" id="cfg-clear-on-kick" data-action="toggleConfigSwitch" data-section="general" data-key="clearOnKick"></div> <span class="config-toggle-label">Clear on Kick ${settingHelpMark('agent-configuration', 'anatomy-of-an-agent', 'Clear on Kick', `Send /clear before each governor kick to reset conversation context. Prevents stale context buildup across runs. On by default for all agents.`)}</span> diff --git a/src/pkg/dashboard/static-src/js/config-state.js b/src/pkg/dashboard/static-src/js/config-state.js index 73897b0141..b5f0cae53e 100644 --- a/src/pkg/dashboard/static-src/js/config-state.js +++ b/src/pkg/dashboard/static-src/js/config-state.js @@ -23,7 +23,7 @@ // which is read-only hub-synced user/role data. It sits next to Model // Gateways because both are backend configuration. const GOVERNOR_BOB_TAB = 'Bob'; - const GOVERNOR_CONFIG_TABS = ['General', 'Appearance', 'Dashboard', 'Agents', 'Thresholds', 'Labels', 'Repos', 'Workflows', 'Budget', 'Notifications', 'Channels', 'Health', 'Sensing', 'Logging', 'Features', 'Extensions', 'Knowledge', 'Project Observability', 'Advisory', 'Smart classifier', 'Work Source', 'Hub', 'Forge App', 'Model Gateways', GOVERNOR_BOB_TAB, 'Variables', 'Security', 'Compliance']; + const GOVERNOR_CONFIG_TABS = ['General', 'Appearance', 'Dashboard', 'Agents', 'Thresholds', 'Labels', 'Repos', 'Workflows', 'Budget', 'Notifications', 'Channels', 'Health', 'Sensing', 'Logging', 'Features', 'Extensions', 'Knowledge', 'Inbox', 'Project Observability', 'Advisory', 'Smart classifier', 'Work Source', 'Hub', 'Forge App', 'Model Gateways', GOVERNOR_BOB_TAB, 'Variables', 'Security', 'Compliance']; const PIPELINE_STAGES = ['resolve-beads', 'track-prs', 'stale-check', 'repo-scan', 'coverage-gate', 'prompt-compose', 'budget-check', 'api-collect', 'final-compose']; const PIPELINE_STAGE_TIPS = { 'resolve-beads': 'Fetch and inject knowledge beads (facts, patterns, gotchas) relevant to the agent\'s current task context.', @@ -86,6 +86,7 @@ { tab: 'Labels', section: 'Labels', label: 'Adoption of a maintainer\'s PR', key: 'governor.labels.stage.adoption.maintainer_policy', text: 'adoption maintainer policy adopt ask never maintainer own pr clanker disconnected offline adopt action governor.labels.stage.adoption.maintainer_policy' }, { tab: 'Labels', section: 'Labels', label: 'Contributor nudge', key: 'governor.labels.stage.contributor_nudge.enabled', text: 'contributor nudge returned-to-contributor red ci fix-pr relay interrupt renudge presence window governor.labels.stage.contributor_nudge.enabled governor.labels.stage.contributor_nudge.renudge_interval auto_merge.nudge_repeat_after governor.labels.stage.contributor_nudge.ci_failure_cooldown governor.labels.stage.presence_window' }, { tab: 'Labels', section: 'Labels', label: 'Adopt red contributor PRs', key: 'governor.labels.stage.adoption.enabled', text: 'adoption adopted-by-hive stage 074 red contributor pr grace online offline supersede takeover in place after offline unregistered heartbeat governor.labels.stage.adoption.enabled governor.labels.stage.contributor_grace_online governor.labels.stage.contributor_grace_offline governor.labels.stage.adoption.after_offline governor.labels.stage.adoption.after_unregistered governor.labels.stage.adoption.in_place governor.labels.stage.adoption.strict_attribution strict attribution governor.labels.stage.adoption.auto_merge auto merge adopted head hive authority governor.labels.stage.adoption.supersede.close_first close first superseded reopen copy failed governor.labels.stage.adoption.close_superseded close superseded on adopt on merge clanker authored opt-in label governor.labels.stage.adoption.offline_grace offline grace rebase failed twice fixer resolving conflicts governor.labels.stage.adoption.rebase_attempt_timeout rebase attempt timeout' }, + { tab: 'Inbox', section: 'Inbox', label: 'Inbox routing', key: 'inbox.spek.repo', lazy: true, text: 'inbox voice recordings plaud wispr default rfc repository spek dibs url auto-routing rules inbox.spek.repo inbox.dibs.url inbox.rules' }, { tab: 'Hub', section: 'Contributor Relay', label: 'Fix budget', key: 'contribute.max_parallel_fixes', text: 'fix budget parallel fixes focus window red pr fix-pr preempt spawn contribute.max_parallel_fixes contribute.min_focus_window' }, { tab: 'Logging', section: 'Logging', label: 'Governor logging', key: 'governor.logging.max_size_mb', text: 'governor logging directory level compression max size age backups governor.logging.compress governor.logging.dir governor.logging.level governor.logging.max_age_days governor.logging.max_backups governor.logging.max_size_mb' }, { tab: 'Advisory', section: 'Advisory', label: 'Advisory digest', key: 'governor.advisory.max_findings', text: 'advisory digest max findings show all staleness pr autoclose target linear issue retry backoff update interval rewrite floor changed digest governor.advisory.update_interval_s governor.advisory.digest_min_interval' }, @@ -130,6 +131,8 @@ { tab: 'Thresholds', section: 'Thresholds', label: 'Delivery SLA targets', key: 'sla.targets', text: 'delivery sla targets sla.targets.start_to_pr_h sla.targets.pr_to_merge_h sla.targets.open_to_merge_h hours sparkline overview stale in-progress' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 connector', key: 'knowledge.connectors', lazy: true, text: 'context7 connector api key env file reference library ids topic validate integrations knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 auto-import', key: 'knowledge.context7.auto_import', lazy: true, text: 'context7 auto import startup libraries knowledge.context7.auto_import[].id knowledge.context7.auto_import[].query' }, + { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Plaud voice notes connector', key: 'knowledge.connectors', lazy: true, text: 'plaud voice notes recordings transcript summary mcp oauth connect npx @plaud-ai/mcp command since_days max_files include_audio tags knowledge.connectors' }, + { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Wispr Flow dictation connector', key: 'knowledge.connectors', lazy: true, text: 'wispr flow wisprflow dictation voice history flow.sqlite sqlite export path since_days max_items min_words group apps include_raw tags knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge pipeline', label: 'Bead synthesizer, curator, code maps and primer', key: 'knowledge.bead_synthesizer.enabled', lazy: true, text: 'knowledge pipeline bead synthesizer wiki facts schedule hourly daily confidence target layer facts per cycle api calls vault retention archive beads high priority dependents curator auto promote threshold promotion from to code maps repository layer max bytes primer facts merge strategy engine public read-only mcp knowledge.engine knowledge.public.enabled knowledge.bead_synthesizer.enabled knowledge.bead_synthesizer.schedule knowledge.bead_synthesizer.min_confidence knowledge.bead_synthesizer.target_layer knowledge.bead_synthesizer.max_facts_per_cycle knowledge.bead_synthesizer.max_api_calls_per_cycle knowledge.bead_synthesizer.vault_path publish to repository knowledge.bead_synthesizer.publish.enabled knowledge.bead_synthesizer.publish.repo knowledge.bead_synthesizer.publish.path knowledge.bead_synthesizer.publish.base_branch knowledge.bead_synthesizer.publish.mode knowledge.bead_synthesizer.publish.cadence knowledge.bead_synthesizer.retention_policy.max_beads knowledge.bead_synthesizer.retention_policy.archive_after_synth_days knowledge.bead_synthesizer.retention_policy.high_priority_retain_days knowledge.bead_synthesizer.retention_policy.preserve_with_deps knowledge.curator.enabled knowledge.curator.schedule knowledge.curator.auto_promote_threshold knowledge.curator.promote_from knowledge.curator.promote_to knowledge.code_maps.enabled knowledge.code_maps.schedule knowledge.code_maps.layer knowledge.code_maps.max_bytes knowledge.code_maps.vault_path knowledge.primer.max_facts knowledge.primer.max_bytes knowledge.primer.merge_strategy' }, { tab: 'Forge App', section: 'Forge App', label: 'GitHub API quota, PR scans and authorship', key: 'github.api_reserve', lazy: true, text: 'github api quota reserve critical agent reserve floor pr detail cache ttl etag cache size mb entries conditional request 304 graphql batch page size app authored prs bot signed commits forge api url docs installation oauth client token github.api_reserve github.api_critical github.agent_reserve_floor github.etag_cache_max_mb github.etag_cache_max_entries github.pr_detail_ttl_s github.graphql_pr_batch github.graphql_pr_batch_page_size github.app_authored_prs github.app_signed_commits github.forge github.api_url github.docs_installation_id github.oauth_client_id github.token github.key_file' }, ]; diff --git a/src/pkg/dashboard/static-src/js/config-tabs.js b/src/pkg/dashboard/static-src/js/config-tabs.js index 24f2b86b70..df9dcef5ff 100644 --- a/src/pkg/dashboard/static-src/js/config-tabs.js +++ b/src/pkg/dashboard/static-src/js/config-tabs.js @@ -56,6 +56,7 @@ if (tabId === 'Features') loadLinearAgentStatus(); // The Features tab also hosts the advisor lane settings panel. if (tabId === 'Features') loadAdvisorSettings(); + if (tabId === 'Inbox') loadInboxSettingsInline(); // The same tab carries the upstream-watch editor and divergence view // (#9969, #11211); it loads from its own endpoint. if (tabId === 'Features') loadUpstreamWatch(); @@ -272,6 +273,7 @@ case 'Features': return renderGovFeatures(d.features || {}, d.auto_merge || {}, d.review || {}); case 'Extensions': return renderGovExtensions(d.features || {}); case 'Knowledge': return renderGovKnowledgeSharing(); + case 'Inbox': return renderGovInbox(); case 'Project Observability': return renderGovProjectObservability(d.project_observability || {}); case 'Advisory': return renderGovAdvisory(d.advisory || {}); case 'Smart classifier': return renderGovSmartClassifier(d.classifier || {}); @@ -297,7 +299,7 @@ } // ---- Settings → Knowledge → Connectors (#11075) ---- - const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'repo-wiki', 'sharepoint']; + const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'plaud', 'repo-wiki', 'sharepoint', 'wisprflow']; const KC_TYPE_INFO = { 'confluence': { label: 'Confluence', description: 'Sync Confluence Cloud or Data Center pages into knowledge facts.' }, 'context7': { label: 'Context7', description: 'Sync documentation for default Context7 libraries. The API key is a reference to an environment variable or secret file; the value is never shown.' }, @@ -306,8 +308,10 @@ 'github-wiki': { label: 'GitHub Wiki', description: 'Clone and index one or more GitHub wiki repositories.' }, 'google-drive': { label: 'Google Drive', description: 'Sync Google Docs, Sheets, markdown, and text files from Drive folders or shared drives.' }, 'notion': { label: 'Notion', description: 'Sync Notion pages and database rows visible to an internal integration.' }, + 'plaud': { label: 'Plaud voice notes', description: 'Sync Plaud Note recordings (Plaud summary + transcript) through the free official Plaud MCP server. Sign-in is Plaud OAuth in your browser; hive never stores the token.' }, 'repo-wiki': { label: 'Repo wiki', description: 'Sync .hive/wiki (or another directory) from one or more GitHub repositories.' }, - 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' } + 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' }, + 'wisprflow': { label: 'Wispr Flow dictation', description: 'Sync your Wispr Flow dictation history (what you spoke, when, into which app) from the desktop app\'s local flow.sqlite or an export folder. Read-only, no sign-in, $0 beyond your Wispr Flow plan.' } }; const KC_TYPE_ICONS = { // Vendor marks: simple-icons (CC0 1.0) paths, 24x24 viewBox, rendered with currentColor. @@ -321,6 +325,8 @@ // Generic glyphs (no vendor): document sheet and open book, drawn in-house. 'context7': 'M4 4h3v16H4zm5 0h3v16H9zm5.2.6 2.9-.8 3.8 15.5-2.9.8z', 'document': 'M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8zm4 18H6V4h7v5h5zM8 12h8v1.6H8zm0 3.2h8v1.6H8zm0 3.2h5V20H8z', + 'plaud': 'M12 2a3.5 3.5 0 0 0-3.5 3.5v6a3.5 3.5 0 0 0 7 0v-6A3.5 3.5 0 0 0 12 2zm-6 9.5h1.8a4.2 4.2 0 0 0 8.4 0H18a6 6 0 0 1-5.1 5.93V20h3v1.8H8.1V20h3v-2.57A6 6 0 0 1 6 11.5z', + 'wisprflow': 'M3 10h2v4H3zm4-3h2v10H7zm4-5h2v20h-2zm4 5h2v10h-2zm4 3h2v4h-2z', 'repo-wiki': 'M4 3h7.2c1.1 0 2 .55 2.8 1.3.8-.75 1.7-1.3 2.8-1.3H20v16h-3.2c-1 0-1.95.4-2.8 1.1V22h-4v-1.9c-.85-.7-1.8-1.1-2.8-1.1H4zm2 2v12h1.2c1.3 0 2.5.4 3.6 1.1V6.2C10.4 5.55 9.6 5 8.8 5zm12 0h-1.2c-.8 0-1.6.55-2 1.2v12.1c1.1-.7 2.3-1.1 3.6-1.1H18z' }; function kbIntegrationIcon(type) { @@ -336,8 +342,10 @@ 'github-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'master']], 'google-drive': [['folder_ids', 'Folder IDs', 'folder123, folder456'], ['shared_drive_ids', 'Shared drive IDs', 'drive123'], ['include_mime', 'Include MIME groups', 'docs,md,txt']], 'notion': [['root_page_ids', 'Root page IDs', '1a2b3c4d5e6f47a8b9c0d1e2f3a4b5c6'], ['database_ids', 'Database IDs', '0f1e2d3c-4b5a-4968-8776-655443322110'], ['include_archived', 'Include archived', 'false'], ['max_pages', 'Max pages', '2000'], ['full_sync_every', 'Full sync every', '10']], + 'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest'], ['since_days', 'Days back on first sync', '30'], ['max_files', 'Max recordings per sync', '200'], ['include_audio', 'Include 24h audio link', 'false'], ['query', 'Name filter (optional)', 'standup'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['home', 'Token store HOME (optional)', '/data/knowledge/connector-state/plaud/home']], 'repo-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'v5'], ['dir', 'Wiki directory', '.hive/wiki']], - 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']] + 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']], + 'wisprflow': [['path', 'flow.sqlite or export folder (empty = app default)', '/Users/me/Library/Application Support/Wispr Flow/flow.sqlite'], ['since_days', 'Days back on first sync', '30'], ['max_items', 'Max dictations per sync', '500'], ['min_words', 'Skip dictations shorter than (words)', '3'], ['group', 'Fact per dictation or day', 'dictation'], ['apps', 'Only these apps (comma-separated, optional)', 'slack, obsidian, vscode'], ['include_raw', 'Include raw ASR text', 'false'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['sqlite3', 'sqlite3 command (optional)', 'sqlite3']] }; let _kcState = { loaded: false, error: '', connectors: [], rows: [], types: [], layers: ['personal', 'project', 'org', 'community'], runtime: false, publish: null }; diff --git a/src/pkg/dashboard/static-src/js/config-watchdog-security.js b/src/pkg/dashboard/static-src/js/config-watchdog-security.js index 24e73ff727..f58c456c12 100644 --- a/src/pkg/dashboard/static-src/js/config-watchdog-security.js +++ b/src/pkg/dashboard/static-src/js/config-watchdog-security.js @@ -541,6 +541,69 @@ markDirty('security', key, n); } + const SUBSTRATE_SANDBOX_FIELDS = [ + { key: 'endpoint', label: 'Endpoint', kind: 'text', global: 'agent_sandbox.substrate.endpoint', agent: 'agents.<key>.sandbox.substrate.endpoint', help: `Substrate Control API address (host:port or an http(s):// URL). Empty uses api.ate-system.svc:443.` }, + { key: 'atespace', label: 'Atespace', kind: 'text', global: 'agent_sandbox.substrate.atespace', agent: 'agents.<key>.sandbox.substrate.atespace', help: `Substrate atespace (its tenant namespace, not a Kubernetes namespace) actors are created in. Empty uses hive.` }, + { key: 'token_env', label: 'Token env', kind: 'text', global: 'agent_sandbox.substrate.token_env', agent: 'agents.<key>.sandbox.substrate.token_env', help: `Environment variable holding a Control API bearer token. Empty reads the token file instead.` }, + { key: 'token_file', label: 'Token file', kind: 'text', global: 'agent_sandbox.substrate.token_file', agent: 'agents.<key>.sandbox.substrate.token_file', help: `Projected ServiceAccount token file hive authenticates with. Empty uses /var/run/secrets/ate/token.` }, + { key: 'ca_file', label: 'CA file', kind: 'text', global: 'agent_sandbox.substrate.ca_file', agent: 'agents.<key>.sandbox.substrate.ca_file', help: `PEM bundle the Control API serving certificate chains to. Empty uses the projected CA when present, else the system roots.` }, + { key: 'insecure_tls', label: 'Insecure TLS', kind: 'bool', global: 'agent_sandbox.substrate.insecure_tls', agent: 'agents.<key>.sandbox.substrate.insecure_tls', help: `Skip Control API and router certificate verification. Dev only.` }, + { key: 'timeout_s', label: 'Timeout (seconds)', kind: 'int', global: 'agent_sandbox.substrate.timeout_s', agent: 'agents.<key>.sandbox.substrate.timeout_s', help: `Bounds one actor from create to delete. 0 uses the sandbox timeout.` }, + { key: 'router_url', label: 'Router URL', kind: 'text', global: 'agent_sandbox.substrate.router_url', agent: 'agents.<key>.sandbox.substrate.router_url', help: `atenet-router CONNECT listener hive reaches the actor bridge through. Empty uses the in-cluster default.` }, + { key: 'router_client_bundle', label: 'Router client bundle', kind: 'text', global: 'agent_sandbox.substrate.router_client_bundle', agent: 'agents.<key>.sandbox.substrate.router_client_bundle', help: `podCertificate bundle the router authenticates hive with. Empty uses the default path.` }, + { key: 'router_ca_file', label: 'Router CA file', kind: 'text', global: 'agent_sandbox.substrate.router_ca_file', agent: 'agents.<key>.sandbox.substrate.router_ca_file', help: `CA the router serving certificate chains to. Empty uses the default path.` }, + { key: 'bridge_port', label: 'Bridge port', kind: 'int', global: 'agent_sandbox.substrate.bridge_port', agent: 'agents.<key>.sandbox.substrate.bridge_port', help: `Port the in-actor bridge listens on. 0 uses 8477.` }, + { key: 'image', label: 'Image', kind: 'text', global: 'agent_sandbox.substrate.image', agent: 'agents.<key>.sandbox.substrate.image', help: `Digest-pinned sandbox image (…@sha256:…) actors run. Substrate refuses tag-only references.` }, + { key: 'bridge_command', label: 'Bridge command', kind: 'lines', global: 'agent_sandbox.substrate.bridge_command[]', agent: 'agents.<key>.sandbox.substrate.bridge_command[]', help: `Actor entrypoint that serves the bridge. One argument per line; empty uses the crux substrate-bridge subcommand.` }, + { key: 'cpu_millicores', label: 'CPU (millicores)', kind: 'int', global: 'agent_sandbox.substrate.cpu_millicores', agent: 'agents.<key>.sandbox.substrate.cpu_millicores', help: `ActorTemplate CPU limit. 0 uses 2000.` }, + { key: 'memory_bytes', label: 'Memory (bytes)', kind: 'int', global: 'agent_sandbox.substrate.memory_bytes', agent: 'agents.<key>.sandbox.substrate.memory_bytes', help: `ActorTemplate memory limit. 0 uses 4 GiB.` }, + { key: 'worker_selector', label: 'Worker selector', kind: 'map', global: 'agent_sandbox.substrate.worker_selector.<key>', agent: 'agents.<key>.sandbox.substrate.worker_selector.<key>', help: `Pins actors to WorkerPools carrying these labels, one key=value per line. Empty means any pool.` }, + { key: 'snapshot_on_idle', label: 'Snapshot on idle', kind: 'bool', global: 'agent_sandbox.substrate.snapshot_on_idle', agent: 'agents.<key>.sandbox.substrate.snapshot_on_idle', help: `Suspend the actor with a snapshot after the kick instead of deleting it. Only the latest snapshot per agent is kept while hive runs (the previous one is deleted on the next suspend); each costs object storage.` }, + { key: 'allow_hosts', label: 'Allow hosts', kind: 'lines', global: 'agent_sandbox.substrate.allow_hosts[]', agent: 'agents.<key>.sandbox.substrate.allow_hosts[]', help: `Additional egress hosts (host or host:port, leading *. wildcard allowed), one per line, on top of the GitHub hosts.` }, + { key: 'policy_profile', label: 'Policy profile', kind: 'text', global: 'agent_sandbox.substrate.policy_profile', agent: 'agents.<key>.sandbox.substrate.policy_profile', help: `Operator-side policy profile recorded on the template.` }, + ]; + + /* substrateSandboxFields renders the Substrate runtime block for either the + global agent_sandbox.substrate (section 'security') or one agent's + sandbox.substrate override (section 'general'). */ + function substrateSandboxFields(section, sub) { + const dirtyKey = section === 'security' ? 'agentSandboxSubstrate' : 'sandboxSubstrate'; + sub = sub || {}; + return SUBSTRATE_SANDBOX_FIELDS.map(f => { + const yaml = section === 'security' ? f.global : f.agent; + const label = 'Substrate ' + f.label.toLowerCase(); + const head = '<div class="config-field"><label>' + esc(label) + ' ' + yamlHint(yaml) + ' ' + settingHelpMark('settings', 'security-and-access', label, esc(f.help)) + '</label>'; + const attrs = 'data-arg0="' + section + '" data-arg1="' + dirtyKey + '" data-arg2="' + f.key + '" data-arg3="' + f.kind + '"'; + const v = sub[f.key]; + if (f.kind === 'bool') { + return head + '<input type="checkbox"' + (v ? ' checked' : '') + ' data-change-action="markSubstrateSandbox" ' + attrs + ' data-arg-types="s,s,s,s,c"></div>'; + } + if (f.kind === 'int') { + return head + '<input type="number" min="0" value="' + Number(v || 0) + '" data-change-action="markSubstrateSandbox" ' + attrs + ' data-arg-types="s,s,s,s,v"></div>'; + } + if (f.kind === 'lines' || f.kind === 'map') { + const text = f.kind === 'map' + ? Object.keys(v || {}).sort().map(k => k + '=' + v[k]).join('\n') + : (v || []).join('\n'); + return head + '<textarea rows="3" class="mono" data-change-action="markSubstrateSandbox" ' + attrs + ' data-arg-types="s,s,s,s,v">' + esc(text) + '</textarea></div>'; + } + return head + '<input type="text" value="' + esc(v || '') + '" data-change-action="markSubstrateSandbox" ' + attrs + ' data-arg-types="s,s,s,s,v"></div>'; + }).join('\n'); + } + + function markSubstrateSandbox(section, dirtyKey, field, kind, value) { + const pending = _configState.dirty[section] && _configState.dirty[section][dirtyKey]; + const base = pending || ((_configState.data && _configState.data[section]) || {})[dirtyKey] || {}; + const next = Object.assign({}, base); + if (kind === 'bool') next[field] = !!value; + else if (kind === 'int') next[field] = Math.max(0, Math.floor(Number(value) || 0)); + else if (kind === 'lines') next[field] = String(value || '').split(/\r?\n/).map(s => s.trim()).filter(Boolean); + else if (kind === 'map') next[field] = parseSettingsMap(value); + else next[field] = String(value || '').trim(); + if (_configState.data && _configState.data[section]) _configState.data[section][dirtyKey] = next; + markDirty(section, dirtyKey, next); + } + function parseSecurityList(value) { return String(value || '').split(',').map(s => s.trim()).filter(Boolean); } @@ -963,6 +1026,9 @@ <div class="config-field"><label>Job TTL (seconds) ${yamlHint('agent_sandbox.job.ttl_seconds')} ${settingHelpMark('settings', 'security-and-access', 'Job TTL (seconds)', `Seconds a finished sandbox Job is kept. 0 uses the default.`)}</label><input type="number" min="0" value="${Number(sec.agentSandboxJobTtlSeconds || 0)}" data-change-action="markSecurityNumber" data-arg0="agentSandboxJobTtlSeconds" data-arg-types="s,v"></div> <div class="config-field"><label>Job workspace claim ${yamlHint('agent_sandbox.job.workspace_claim')} ${settingHelpMark('settings', 'security-and-access', 'Job workspace claim', `PVC holding the workspace for the job runtime. Required for runtime job.`)}</label><input type="text" value="${esc(sec.agentSandboxJobWorkspaceClaim || '')}" placeholder="PersistentVolumeClaim name" data-change-action="markDirty" data-arg0="security" data-arg1="agentSandboxJobWorkspaceClaim" data-arg-types="s,s,v"></div> <div class="config-field"><label>Job workspace claim mount ${yamlHint('agent_sandbox.job.workspace_claim_mount')} ${settingHelpMark('settings', 'security-and-access', 'Job workspace claim mount', `Where the workspace claim is mounted in the Job.`)}</label><input type="text" value="${esc(sec.agentSandboxJobWorkspaceClaimMount || '')}" placeholder="mount path" data-change-action="markDirty" data-arg0="security" data-arg1="agentSandboxJobWorkspaceClaimMount" data-arg-types="s,s,v"></div> + <details class="settings-subsection-details"><summary class="settings-subsection-summary">Substrate sandbox</summary> + ${substrateSandboxFields('security', sec.agentSandboxSubstrate)} + </details> </details>`; /* Backup encryption key: painted asynchronously from presence-only diff --git a/src/pkg/dashboard/static-src/js/inbox.js b/src/pkg/dashboard/static-src/js/inbox.js new file mode 100644 index 0000000000..c638c57712 --- /dev/null +++ b/src/pkg/dashboard/static-src/js/inbox.js @@ -0,0 +1,348 @@ + // ── Inbox: source-agnostic triage of voice recordings and dictations ── + // Items come from knowledge connectors (plaud, wisprflow, …); each one is + // already a Knowledge fact. Routing them to spek (RFC + child issues) or + // dibs happens inside the hive via POST /api/inbox/route. + const INBOX_POLL_MS = 60000; + const INBOX_BUSY_POLL_MS = 5000; + const INBOX_SOURCE_ICONS = { plaud: '🎙️', wisprflow: '🗣️', 'file-drop': '📎', webhook: '🔗' }; + const INBOX_DEST_LABELS = { spek_rfc: 'RFC', dibs: 'dibs' }; + let _inboxCache = null; + let _inboxSelected = new Set(); + let _inboxShowProcessed = false; + let _inboxBusyTimer = null; + + function inboxSourceIcon(src) { return INBOX_SOURCE_ICONS[src] || '📥'; } + function inboxItemKey(it) { return it.source + '/' + it.id; } + function inboxFmtDuration(ms) { + const s = Math.round((Number(ms) || 0) / 1000); + if (s <= 0) return ''; + const m = Math.floor(s / 60); + return m >= 60 ? Math.floor(m / 60) + 'h ' + (m % 60) + 'm' : m + 'm ' + (s % 60) + 's'; + } + function inboxRouteBusy(items) { + return (items || []).some(it => Object.values(it.routes || {}).some(r => r && (r.status === 'queued' || r.status === 'running'))); + } + + async function fetchInbox() { + try { + const r = await fetch('/api/inbox'); + if (!r.ok) { + // Never leave "Loading inbox…" up: a 404/500 becomes a visible error. + const body = await r.json().catch(() => ({})); + _inboxCache = { items: [], sources: [], settings: {}, error: body.error || ('HTTP ' + r.status) }; + renderInbox(); + return; + } + _inboxCache = await r.json(); + renderInbox(); + clearTimeout(_inboxBusyTimer); + if (inboxRouteBusy(_inboxCache.items)) _inboxBusyTimer = setTimeout(fetchInbox, INBOX_BUSY_POLL_MS); + } catch (e) { /* transient — leave last render in place */ } + } + dashboardScheduleWork('inbox', fetchInbox).catch(console.error); + dashboardSetInterval('inbox', fetchInbox, INBOX_POLL_MS); + + function inboxStatePill(dest, st) { + const label = INBOX_DEST_LABELS[dest] || dest; + if (!st) return ''; + const status = st.status === 'filed' || st.status === 'delegated' ? 'ok' : (st.status === 'failed' ? 'error' : 'syncing'); + const text = st.status === 'filed' ? label + ' filed' : st.status === 'delegated' ? label + ' handed to agent' : label + ' ' + st.status; + const title = st.error ? escapeHtml(st.error) : (st.repo ? escapeHtml(st.repo) : ''); + let html = '<span class="kc-pill" data-status="' + status + '" title="' + title + '">' + escapeHtml(text) + '</span>'; + if (st.url) html += ' <a class="inbox-link" href="' + escapeHtml(st.url) + '" target="_blank" rel="noopener">open ↗</a>'; + if (Array.isArray(st.children) && st.children.length) html += ' <span class="muted-xs">' + st.children.length + ' child issue' + (st.children.length === 1 ? '' : 's') + '</span>'; + return html; + } + + function inboxItemHtml(it) { + const key = inboxItemKey(it); + const checked = _inboxSelected.has(key) ? ' checked' : ''; + const when = it.recorded_at && !String(it.recorded_at).startsWith('0001') ? relativeTimeShort(it.recorded_at) : ''; + const dur = inboxFmtDuration(it.duration_ms); + const meta = [escapeHtml(it.source), when, dur].filter(Boolean).join(' · '); + const routes = it.routes || {}; + let pills = ''; + for (const d of Object.keys(routes)) pills += inboxStatePill(d, routes[d]) + ' '; + if (it.dismissed) pills += '<span class="kc-pill" title="Kept as a Knowledge fact only">knowledge only</span>'; + const owner = viewerIsOwner(); + const src = escapeHtml(it.source), id = escapeHtml(it.id); + let actions = ''; + if (owner) { + const retry = (d) => routes[d] && routes[d].status === 'failed' ? 'Retry ' : '→ '; + actions += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="inboxOpenRoute" data-arg0="' + src + '" data-arg1="' + id + '" data-arg2="spek_rfc">' + retry('spek_rfc') + 'RFC…</button>'; + actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenRoute" data-arg0="' + src + '" data-arg1="' + id + '" data-arg2="dibs">' + retry('dibs') + 'dibs…</button>'; + if (!it.dismissed) actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismiss" data-arg0="' + src + '" data-arg1="' + id + '" title="Keep it as a Knowledge fact; no RFC, no dibs idea">Knowledge only</button>'; + else actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxUndismiss" data-arg0="' + src + '" data-arg1="' + id + '">Back to inbox</button>'; + } + if (it.fact_slug) actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenFact" data-arg0="' + escapeHtml(it.fact_slug) + '" data-arg1="' + escapeHtml(it.layer || '') + '" title="This item is already a Knowledge fact">📚 Fact</button>'; + return '<div class="kb-fact-item inbox-item" data-inbox-key="' + escapeHtml(key) + '">' + + (owner ? '<label class="inbox-check"><input type="checkbox" data-change-action="inboxToggleSelect" data-arg0="' + src + '" data-arg1="' + id + '"' + checked + ' aria-label="Select ' + escapeHtml(it.title) + '"></label>' : '') + + '<div class="inbox-icon" title="' + src + '">' + inboxSourceIcon(it.source) + '</div>' + + '<div class="kb-fact-body"><div class="kb-fact-title">' + escapeHtml(it.title || it.id) + '</div>' + + '<div class="muted-xs">' + meta + '</div>' + + (it.excerpt ? '<div class="kb-fact-snippet">' + escapeHtml(it.excerpt) + '</div>' : '') + + (pills ? '<div class="inbox-pills">' + pills + '</div>' : '') + + '<div class="kc-actions mt-2">' + actions + '</div></div></div>'; + } + + function renderInbox() { + refreshSectionCardShell('inbox-section'); + const panel = document.getElementById('inbox-panel'); + const badge = document.getElementById('inbox-sidebar-badge'); + const data = _inboxCache; + if (!panel) return; + if (!data) { panel.innerHTML = '<div class="muted-note">Loading inbox…</div>'; return; } + const items = data.items || []; + const unprocessed = Number(data.unprocessed) || 0; + if (badge) setSidebarCountBadge(badge, unprocessed, unprocessed + ' unprocessed'); + setSectionSummary('inbox-section', unprocessed + ' to triage · ' + items.length + ' total'); + const owner = viewerIsOwner(); + const sources = (data.sources || []).map(s => { + const status = s.connected ? 'ok' : (s.configured ? 'error' : ''); + const label = inboxSourceIcon(s.type) + ' ' + escapeHtml(s.type) + (s.connected ? (s.account ? ': ' + escapeHtml(s.account) : ': connected') : s.configured ? ': not connected' : ': not configured'); + return '<span class="kc-pill" data-status="' + status + '">' + label + '</span>'; + }).join(' '); + const settings = data.settings || {}; + const rules = Array.isArray(settings.rules) ? settings.rules.length : 0; + let html = '<div class="row-between inbox-toolbar">' + + '<div class="inbox-sources">' + (sources || '<span class="muted-xs">No voice sources configured.</span>') + + ' <button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbOpenIntegrations">Connect sources</button></div>' + + '<div class="kc-actions">' + + '<label class="muted-xs inbox-toggle"><input type="checkbox" data-change-action="inboxToggleProcessed"' + (_inboxShowProcessed ? ' checked' : '') + '> show processed</label>' + + (owner ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenSettings" title="Auto-routing rules, default RFC repository, dibs URL">⚙ Auto-routing (' + rules + ')</button>' : '') + + '</div></div>'; + if (data.error) html += '<div class="kc-error muted-note">' + escapeHtml(data.error) + '</div>'; + if (!settings.spek_ready && owner) html += '<p class="muted-xs">RFC filing needs a <code>spektacular</code> binary on the hive host (<code>inbox.spek.binary</code>) or at least one enabled agent to hand the job to (<code>inbox.spek.agent</code> overrides the default).</p>'; + const shown = items.filter(it => _inboxShowProcessed || !it.processed); + if (owner && shown.length) { + const sel = _inboxSelected.size; + html += '<div class="kc-actions mb-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxSelectAll">' + (sel === shown.length ? 'Clear selection' : 'Select all') + '</button>' + + (sel ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismissSelected">Knowledge only (' + sel + ')</button>' : '') + '</div>'; + } + if (!shown.length) html += '<div class="muted-note">' + (items.length ? 'Everything is triaged. Toggle “show processed” to see filed and dismissed items.' : 'Nothing yet. Connect Plaud or another voice source under Knowledge → Integrations; new recordings land here after the next sync (a newly added connector starts on the next hive restart).') + '</div>'; + else html += '<div class="kb-fact-list inbox-list">' + shown.map(inboxItemHtml).join('') + '</div>'; + panel.innerHTML = html; + } + + function inboxToggleProcessed() { _inboxShowProcessed = !_inboxShowProcessed; renderInbox(); } + function inboxToggleSelect(src, id) { + const key = src + '/' + id; + if (_inboxSelected.has(key)) _inboxSelected.delete(key); else _inboxSelected.add(key); + renderInbox(); + } + function inboxSelectAll() { + const shown = ((_inboxCache || {}).items || []).filter(it => _inboxShowProcessed || !it.processed); + if (_inboxSelected.size === shown.length) _inboxSelected.clear(); + else _inboxSelected = new Set(shown.map(inboxItemKey)); + renderInbox(); + } + function inboxFind(src, id) { return ((_inboxCache || {}).items || []).find(it => it.source === src && it.id === id); } + function inboxOpenFact(slug, layer) { + ocNavigate('knowledge-section'); + kbReadFact(slug, layer || undefined); + } + + async function inboxPostDismiss(items, undo) { + const r = await fetch('/api/inbox/dismiss', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ items, undo: !!undo }) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + return data; + } + async function inboxDismiss(src, id) { + try { await inboxPostDismiss([{ source: src, item_id: id }], false); showToast('Kept as a Knowledge fact only', 'success'); await fetchInbox(); } + catch (e) { showToast('Dismiss failed: ' + e.message, 'error'); } + } + async function inboxUndismiss(src, id) { + try { await inboxPostDismiss([{ source: src, item_id: id }], true); await fetchInbox(); } + catch (e) { showToast('Undo failed: ' + e.message, 'error'); } + } + async function inboxDismissSelected() { + const items = Array.from(_inboxSelected).map(k => { const i = k.indexOf('/'); return { source: k.slice(0, i), item_id: k.slice(i + 1) }; }); + if (!items.length) return; + if (!(await hiveConfirm('Mark ' + items.length + ' item' + (items.length === 1 ? '' : 's') + ' as Knowledge only? They stay searchable as facts; no RFC or dibs idea is created.', 'Knowledge only'))) return; + try { await inboxPostDismiss(items, false); _inboxSelected.clear(); await fetchInbox(); } + catch (e) { showToast('Dismiss failed: ' + e.message, 'error'); } + } + + // Route dialog: destination-specific confirm with a repo picker for RFCs. + function inboxOpenRoute(src, id, dest) { + const it = inboxFind(src, id); + if (!it) return; + const settings = (_inboxCache || {}).settings || {}; + const repos = (_inboxCache || {}).repos || []; + const def = settings.spek_repo || repos[0] || ''; + let body = '<p class="muted-note m-0"><strong>' + escapeHtml(it.title || it.id) + '</strong></p>'; + if (dest === 'spek_rfc') { + const opts = repos.map(r => '<option value="' + escapeHtml(r) + '"' + (r === def ? ' selected' : '') + '>' + escapeHtml(r) + '</option>').join(''); + body += '<p class="muted-note">A hive-run <code>spektacular voice</code> turns the transcript into a spec, then files one RFC issue plus child issues on the chosen repository. The children are picked up by the hive like any other issue.</p>' + + '<label class="muted-note inbox-field">Repository<select class="field-input field-input-block mt-2" id="inbox-route-repo">' + opts + '</select></label>' + + (repos.length ? '' : '<p class="kc-error muted-xs">No repositories configured on this hive.</p>'); + } else { + body += '<p class="muted-note">Posts the title, summary and transcript to dibs as a private draft idea' + (settings.dibs_url ? ' at <code>' + escapeHtml(settings.dibs_url) + '</code>' : '') + '. Provenance links back to <code>' + escapeHtml(src) + '://recordings/' + escapeHtml(id) + '</code>.</p>' + + (!settings.dibs_url ? '<p class="kc-error muted-xs">dibs is not configured — set the URL under ⚙ Auto-routing and <code>inbox.dibs.token_env</code> in the hive config.</p>' : !settings.dibs_token_set ? '<p class="kc-error muted-xs">dibs token missing — set the environment variable named by <code>inbox.dibs.token_env</code>.</p>' : ''); + } + body += '<div class="hive-dialog-error" role="alert" id="inbox-route-error"></div>' + + '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxConfirmRoute" data-arg0="' + escapeHtml(src) + '" data-arg1="' + escapeHtml(id) + '" data-arg2="' + escapeHtml(dest) + '">' + (dest === 'spek_rfc' ? 'File RFC' : 'Send to dibs') + '</button></div>'; + kbShowModal(dest === 'spek_rfc' ? '→ RFC (spektacular)' : '→ dibs', body, { width: '520px' }); + } + async function inboxConfirmRoute(src, id, dest) { + const repoEl = document.getElementById('inbox-route-repo'); + const repo = repoEl ? repoEl.value : ''; + const errEl = document.getElementById('inbox-route-error'); + try { + const r = await fetch('/api/inbox/route', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ source: src, item_id: id, destination: dest, repo }) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + kbCloseModal(); + showToast((INBOX_DEST_LABELS[dest] || dest) + ' queued — the pill updates when it finishes.', 'success'); + await fetchInbox(); + } catch (e) { + if (errEl) errEl.textContent = e.message; else showToast('Route failed: ' + e.message, 'error'); + } + } + + // Settings drawer: default RFC repo, dibs URL, and ordered auto-routing rules. + let _inboxDraftRules = []; + function inboxRuleRowHtml(rule, i) { + const sources = ((_inboxCache || {}).settings || {}).sources || []; + const srcOpts = ['<option value=""' + (!rule.source ? ' selected' : '') + '>any source</option>'].concat(sources.map(s => '<option value="' + escapeHtml(s) + '"' + (rule.source === s ? ' selected' : '') + '>' + escapeHtml(s) + '</option>')).join(''); + const destOpts = [['spek_rfc', '→ RFC'], ['dibs', '→ dibs'], ['dismiss', 'Knowledge only']].map(([v, l]) => '<option value="' + v + '"' + (rule.destination === v ? ' selected' : '') + '>' + l + '</option>').join(''); + return '<div class="inbox-rule" data-rule-index="' + i + '">' + + '<select class="field-input" data-rule-field="source" aria-label="Source">' + srcOpts + '</select>' + + '<input class="kb-search-box" data-rule-field="title_prefix" placeholder="title prefix (RFC:)" value="' + escapeHtml(rule.title_prefix || '') + '" aria-label="Title prefix">' + + '<input class="kb-search-box" data-rule-field="tag" placeholder="or tag" value="' + escapeHtml(rule.tag || '') + '" aria-label="Tag">' + + '<select class="field-input" data-rule-field="destination" aria-label="Destination">' + destOpts + '</select>' + + '<input class="kb-search-box" data-rule-field="repo" placeholder="owner/repo (RFC only)" value="' + escapeHtml(rule.repo || '') + '" aria-label="Repository">' + + '<button type="button" class="hv-btn btn-icon btn-sm" data-action="inboxRemoveRule" data-arg0="' + i + '" aria-label="Remove rule">✕</button></div>'; + } + // The same form renders in the Inbox page modal and inline under + // Settings → Inbox, and both can sit in the DOM at once (the settings + // tab stays mounted; the closed modal keeps its last body). Every read + // and write therefore goes through the active form root: the modal + // while it is open, otherwise the inline section. + function inboxSettingsRoot() { + const overlay = document.getElementById('kb-modal-overlay'); + if (overlay && overlay.style.display !== 'none') { + const modal = overlay.querySelector('.inbox-settings-form'); + if (modal) return modal; + } + return document.querySelector('#inbox-settings-inline .inbox-settings-form'); + } + function inboxSettingsField(root, name) { + return root ? root.querySelector('[data-inbox-field="' + name + '"]') : null; + } + function inboxReadRules(root) { + root = root || inboxSettingsRoot(); + if (!root) return []; + return Array.from(root.querySelectorAll('.inbox-settings-rules .inbox-rule')).map(row => { + const v = f => { const el = row.querySelector('[data-rule-field="' + f + '"]'); return el ? el.value.trim() : ''; }; + return { source: v('source'), title_prefix: v('title_prefix'), tag: v('tag'), destination: v('destination'), repo: v('repo') }; + }); + } + function inboxRenderRules(root) { + root = root || inboxSettingsRoot(); + const host = root ? root.querySelector('.inbox-settings-rules') : null; + if (!host) return; + host.innerHTML = _inboxDraftRules.length ? _inboxDraftRules.map(inboxRuleRowHtml).join('') : '<p class="muted-xs">No rules — every item waits here for manual triage.</p>'; + } + function inboxSettingsFormHtml(settings, inline) { + const buttons = inline + ? '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>' + : '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>'; + return '<div class="inbox-settings-form" data-inbox-form="' + (inline ? 'inline' : 'modal') + '"><p class="muted-note m-0">New items are checked against these rules in order after every connector sync; the first match routes the item automatically. Items that arrived before a rule existed are never re-routed.</p>' + + '<div class="inbox-settings-grid">' + + '<label class="muted-note inbox-field">Default RFC repository<input class="kb-search-box mt-2" type="text" data-inbox-field="spek-repo" placeholder="owner/repo" autocomplete="off" value="' + escapeHtml(settings.spek_repo || '') + '"></label>' + + '<label class="muted-note inbox-field">dibs URL<input class="kb-search-box mt-2" type="text" data-inbox-field="dibs-url" placeholder="https://dibs.example.org" autocomplete="off" value="' + escapeHtml(settings.dibs_url || '') + '"></label></div>' + + '<p class="muted-xs">The dibs token is read from the environment variable named by <code>inbox.dibs.token_env</code>' + (settings.dibs_token_set ? ' (set)' : ' (not set)') + '.</p>' + + '<div class="row-between mt-4 mb-2"><div class="section-label m-0">Rules</div><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxAddRule">Add rule</button></div>' + + '<div class="inbox-settings-rules"></div>' + + '<div class="hive-dialog-error" role="alert" data-inbox-field="error"></div>' + buttons + '</div>'; + } + function inboxOpenSettings() { + const settings = (_inboxCache || {}).settings || {}; + _inboxDraftRules = (settings.rules || []).map(r => Object.assign({}, r)); + kbShowModal('⚙ Inbox auto-routing', inboxSettingsFormHtml(settings, false), { width: '720px' }); + inboxRenderRules(); + } + // Settings → Inbox: the same form inline, loaded from /api/inbox/settings. + function renderGovInbox() { + return '<div class="config-section" data-config-section="Inbox"><h3>Inbox</h3>' + + '<p class="text-muted fs-base">Default RFC repository, dibs URL and auto-routing rules for voice recordings. ' + yamlHint('inbox.spek.repo') + ' ' + yamlHint('inbox.dibs.url') + ' ' + yamlHint('inbox.rules') + '</p>' + + '<div id="inbox-settings-inline"><p class="muted-xs">Loading…</p></div></div>'; + } + async function loadInboxSettingsInline() { + const host = document.getElementById('inbox-settings-inline'); + if (!host) return; + try { + const r = await fetch('/api/inbox/settings'); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _inboxCache = Object.assign({}, _inboxCache, { settings: data }); + _inboxDraftRules = (data.rules || []).map(x => Object.assign({}, x)); + host.innerHTML = inboxSettingsFormHtml(data, true); + inboxRenderRules(host.querySelector('.inbox-settings-form')); + } catch (e) { + host.innerHTML = '<p class="kc-error muted-xs">Inbox settings unavailable: ' + escapeHtml(e.message) + '</p>'; + } + } + function inboxAddRule() { + _inboxDraftRules = inboxReadRules(); + _inboxDraftRules.push({ source: '', title_prefix: 'RFC:', tag: '', destination: 'spek_rfc', repo: '' }); + inboxRenderRules(); + } + function inboxRemoveRule(i) { + _inboxDraftRules = inboxReadRules(); + _inboxDraftRules.splice(Number(i), 1); + inboxRenderRules(); + } + async function inboxSaveSettings() { + const root = inboxSettingsRoot(); + if (!root) return; + const errEl = inboxSettingsField(root, 'error'); + const rules = inboxReadRules(root).map(r => { const o = {}; for (const k of Object.keys(r)) if (r[k]) o[k] = r[k]; return o; }); + const payload = { rules, spek_repo: (inboxSettingsField(root, 'spek-repo') || {}).value || '', dibs_url: (inboxSettingsField(root, 'dibs-url') || {}).value || '' }; + try { + const r = await fetch('/api/inbox/settings', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + if (root.dataset.inboxForm !== 'inline') kbCloseModal(); + showToast('Inbox settings saved', 'success'); + await fetchInbox(); + } catch (e) { + if (errEl) errEl.textContent = e.message; else showToast('Save failed: ' + e.message, 'error'); + } + } + + // Hosted-hive Plaud sign-in completion: Plaud's OAuth client only redirects + // to localhost:8199, so the operator pastes either the redirect URL the + // browser landed on or the tokens-mcp.json contents; the hive finishes + // the login in-pod. + async function kbPlaudPasteComplete(name) { + const el = document.getElementById('kb-plaud-paste-' + name); + const raw = el ? el.value.trim() : ''; + if (!raw) { showToast('Paste the redirect URL or the tokens JSON first.', 'error'); return; } + const body = raw.startsWith('{') ? { tokens: raw } : { redirect_url: raw }; + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/complete', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + showToast(data.connected ? 'Plaud connected' + (data.account ? ' as ' + data.account : '') : 'Plaud sign-in submitted; click Check.', data.connected ? 'success' : 'info'); + fetchInbox(); + } catch (e) { + showToast('Plaud sign-in failed: ' + e.message, 'error'); + } + } + function kbPlaudPasteHtml(name) { + const n = escapeHtml(name); + const a = _kbPlaudAuth[name] || {}; + const openInbox = '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="ocNavigate" data-arg0="inbox-section">📥 Open Inbox</button>'; + if (a.connected) return '<div class="kc-actions mt-2">' + openInbox + '<span class="muted-xs">Recordings are triaged in the Inbox, not here.</span></div>'; + return '<details class="kb-plaud-paste mt-2"><summary class="muted-xs">Hive runs on a server? Finish sign-in by pasting</summary>' + + '<p class="muted-xs">Plaud only redirects to <code>localhost:8199</code>. Open the authorization link, approve, then paste the <code>localhost:8199/auth/callback?…</code> URL your browser lands on (within 10 minutes). Alternatively paste the contents of <code>~/.plaud/tokens-mcp.json</code> from a laptop where <code>plaud-mcp</code> is signed in.</p>' + + '<textarea class="kb-search-box" id="kb-plaud-paste-' + n + '" rows="3" placeholder="http://localhost:8199/auth/callback?code=… or {"access_token": …}"></textarea>' + + '<div class="kc-actions mt-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudPasteComplete" data-arg0="' + n + '">Submit</button>' + openInbox + '</div></details>'; + } diff --git a/src/pkg/dashboard/static-src/js/knowledge.js b/src/pkg/dashboard/static-src/js/knowledge.js index b42fcb2c14..1e08331dc8 100644 --- a/src/pkg/dashboard/static-src/js/knowledge.js +++ b/src/pkg/dashboard/static-src/js/knowledge.js @@ -2004,6 +2004,18 @@ html += '<button class="hv-btn btn-primary nous-btn btn-blue" data-action="kbImportDocFromModal">Import</button>'; html += '</div></div></details>'; + html += '<details class="kb-sources-add"><summary class="summary-label">🎙️ Plaud voice notes</summary>'; + html += '<div class="stack kb-sources-form">'; + html += '<div class="muted-xs">Make your Plaud Note recordings (Plaud summary + transcript) searchable knowledge via the free official Plaud MCP server. $0 beyond your Plaud plan; sign-in is Plaud OAuth in your browser.</div>'; + html += '<div><button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbOpenPlaudIntegration">Configure in Integrations</button></div>'; + html += '</div></details>'; + + html += '<details class="kb-sources-add"><summary class="summary-label">🗣️ Wispr Flow dictation</summary>'; + html += '<div class="stack kb-sources-form">'; + html += '<div class="muted-xs">Dictate straight into any field here — put the cursor in <strong>+ New Fact</strong> and hold the Flow key; no connector needed. To also pull your past dictations in, the Wispr Flow connector reads the app\'s local history (flow.sqlite or an export folder). $0 beyond your Wispr Flow plan.</div>'; + html += '<div><button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbOpenWisprFlowIntegration">Configure in Integrations</button></div>'; + html += '</div></details>'; + html += '<details open class="kb-sources-add"><summary class="summary-label">📚 Import from Context7</summary>'; html += '<div class="stack kb-sources-form">'; html += '<div class="muted-xs">Search Context7 for library documentation (e.g. "vllm", "next.js", "kubernetes").</div>'; @@ -2485,9 +2497,12 @@ html += '<div class="kb-integration-connector-title"><code>' + escapeHtml(c.name) + '</code> <span class="kc-pill" data-status="' + escapeHtml(status) + '">' + escapeHtml(status) + '</span></div>'; html += '<div class="muted-xs">' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '</div>'; if (r.last_error) html += '<div class="kc-error" title="' + escapeHtml(r.last_error) + '">' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '</div>'; + if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name); + if (type === 'wisprflow') html += kbWisprFlowSourceLine(c); html += '</div>'; if (owner) { html += '<div class="kb-integration-actions">'; + if (type === 'plaud') html += kbPlaudAuthButtons(c.name); html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationSync" data-arg0="' + escapeHtml(c.name) + '"' + (status === 'syncing' ? ' disabled' : '') + '>Sync now</button>'; html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationEdit" data-arg0="' + escapeHtml(c.name) + '">Edit</button>'; html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationToggle" data-arg0="' + escapeHtml(c.name) + '">' + (c.enabled ? 'Disable' : 'Enable') + '</button>'; @@ -2503,7 +2518,7 @@ const connectors = (_kcState.connectors || []).filter(c => c.type === type); if (!_kbIntegrationEditName || !connectors.some(c => c.name === _kbIntegrationEditName)) _kbIntegrationEditName = connectors[0]?.name || ''; const edited = connectors.find(c => c.name === _kbIntegrationEditName) || { name: '', type, enabled: true, layer: 'project', interval: '', scope: {}, auth: {} }; - let html = '<div class="card-inset mb-5"><div class="row-between"><div><div class="title-sm kb-integration-title">' + kbIntegrationIcon(type) + escapeHtml(info.label) + '</div><p class="muted-note m-0">' + escapeHtml(info.description) + '</p></div><a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Docs ↗</a></div></div>'; + let html = '<div class="card-inset mb-5"><div class="row-between"><div><div class="title-sm kb-integration-title">' + kbIntegrationIcon(type) + escapeHtml(info.label) + '</div><p class="muted-note m-0">' + escapeHtml(info.description) + '</p></div><span class="row">' + (type === 'plaud' ? '<a href="/docs/inbox-voice-sources.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Inbox quickstart ↗</a>' : '') + '<a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Docs ↗</a></span></div></div>'; html += '<div class="row-between mb-4"><div class="section-label m-0">Configured entries</div>' + (owner ? '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbIntegrationNew" data-arg0="' + escapeHtml(type) + '">Add ' + escapeHtml(info.label) + '</button>' : '') + '</div>'; html += connectors.length ? connectors.map(c => kbConnectorRow(type, c, owner)).join('') : '<div class="empty-dashed mb-4">No ' + escapeHtml(info.label) + ' connector configured yet.</div>'; html += owner ? kbConnectorFormHTML(type, edited, !!edited.name) : '<p class="muted-note">Only the hive owner can edit connector configuration.</p>'; @@ -2524,6 +2539,8 @@ html += '<label class="muted-note">Credential reference<input class="kb-search-box mt-2" type="text" data-kb-kc-auth-value value="' + escapeHtml(authVal) + '" placeholder="TOKEN_ENV or /run/secrets/token" autocomplete="off"></label></div>'; html += '<div class="kb-integration-scope-grid">' + kcScopeFieldsHTML(type, c.scope || {}) + '</div>'; if (type === 'context7') html += kbContext7FormExtras(c); + if (type === 'wisprflow') html += '<p class="muted-xs" data-kb-wisprflow-note>Leave the credential source as None: the connector reads Wispr Flow\'s local history read-only with the sqlite3 CLI. Leave the path empty for the default app location on the hive host (macOS: ~/Library/Application Support/Wispr Flow/flow.sqlite; Windows: %APPDATA%\\Wispr Flow\\flow.sqlite), or point it at a copy of flow.sqlite or a folder of exported .json/.ndjson/.csv/.md files. New dictations need no connector at all — dictate into + New Fact directly.</p>'; + if (type === 'plaud') html += '<p class="muted-xs" data-kb-plaud-note>Leave the credential source as None: Plaud signs in with its own OAuth flow (Connect on the entry above) and keeps the token in the MCP server\'s own store. Requires Node.js ≥ 20 on the hive host and costs nothing beyond your Plaud plan.</p>'; html += '<label class="muted-note kb-integration-enabled"><input type="checkbox" data-kb-kc-enabled' + (c.enabled ? ' checked' : '') + '> Enabled</label>'; html += '<div class="hive-dialog-error" role="alert" data-kb-kc-msg></div>'; html += '<div class="row-end kb-integration-form-actions"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationValidate">Validate</button><button type="button" class="hv-btn btn-primary btn-sm" data-action="kbIntegrationSave">Save</button></div></div>'; @@ -2607,3 +2624,91 @@ await kcSyncNow(name); setTimeout(() => kbLoadIntegrationsModal(_kbIntegrationSelected), 1500); } + + // ── Plaud sign-in (OAuth through the Plaud MCP server) ── + // Cached per connector name; loaded lazily when the Plaud card renders. + const _kbPlaudAuth = {}; + function kbPlaudAuthLine(name) { + const a = _kbPlaudAuth[name]; + if (!a) { + setTimeout(() => kbPlaudAuthLoad(name, false), 0); + return '<div class="muted-xs" data-kb-plaud-auth="' + escapeHtml(name) + '">Plaud: checking sign-in…</div>'; + } + let text; + if (a.error) text = 'Plaud: ' + a.error; + else if (a.pending) text = 'Plaud: waiting for authorization' + (a.auth_url ? ' — <a href="' + escapeHtml(a.auth_url) + '" target="_blank" rel="noopener">open the Plaud authorization page ↗</a>' : ' (URL pending…)'); + else if (a.connected) text = 'Plaud: connected' + (a.account ? ' as <strong>' + escapeHtml(a.account) + '</strong>' : ''); + else text = 'Plaud: not connected' + (a.message ? ' — ' + escapeHtml(a.message) : ''); + return '<div class="muted-xs" data-kb-plaud-auth="' + escapeHtml(name) + '" data-connected="' + (a.connected ? '1' : '0') + '">' + text + '</div>'; + } + function kbPlaudAuthButtons(name) { + const a = _kbPlaudAuth[name] || {}; + const n = escapeHtml(name); + let html = ''; + if (a.connected) html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudDisconnect" data-arg0="' + n + '">Disconnect</button>'; + else html += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbPlaudConnect" data-arg0="' + n + '"' + (a.pending ? ' disabled' : '') + '>Connect Plaud</button>'; + html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudRefreshAuth" data-arg0="' + n + '" title="Re-check sign-in with Plaud">Check</button>'; + return html; + } + async function kbPlaudAuthLoad(name, refresh) { + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth' + (refresh ? '?refresh=1' : '')); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + } catch (e) { + _kbPlaudAuth[name] = { error: e.message }; + } + if (document.querySelector('[data-kb-plaud-auth="' + CSS.escape(name) + '"]')) kbRenderIntegrationsModal(); + } + async function kbPlaudRefreshAuth(name) { + showToast('Checking Plaud sign-in…', 'info'); + await kbPlaudAuthLoad(name, true); + } + async function kbPlaudConnect(name) { + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/connect', { method: 'POST' }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + if (data.connected) showToast('Plaud connected' + (data.account ? ' as ' + data.account : ''), 'success'); + else if (data.auth_url) { showToast('Open the Plaud authorization link shown on the card, then click Check.', 'info'); kbPlaudPollAuth(name, 0); } + else showToast('Plaud login started; the authorization URL will appear on the card shortly.', 'info'); + if (!data.connected && !data.auth_url) kbPlaudPollAuth(name, 0); + } catch (e) { + showToast('Plaud connect failed: ' + e.message, 'error'); + } + } + // Poll the cached state while a login is pending (URL arrival, completion). + function kbPlaudPollAuth(name, attempt) { + if (attempt > 60) return; + setTimeout(async () => { + await kbPlaudAuthLoad(name, false); + const a = _kbPlaudAuth[name] || {}; + if (a.connected) { showToast('Plaud connected' + (a.account ? ' as ' + a.account : ''), 'success'); return; } + if (a.pending) kbPlaudPollAuth(name, attempt + 1); + }, 5000); + } + async function kbPlaudDisconnect(name) { + if (!(await hiveConfirm('Sign out of Plaud? Synced recordings stay in the vault until you remove the connector.'))) return; + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/disconnect', { method: 'POST' }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + showToast('Plaud disconnected', 'success'); + } catch (e) { + showToast('Plaud disconnect failed: ' + e.message, 'error'); + } + } + function kbOpenPlaudIntegration() { _kbIntegrationSelected = 'plaud'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } + + // ── Wispr Flow (local dictation history, no sign-in) ── + function kbWisprFlowSourceLine(c) { + const path = (c.scope && c.scope.path) || ''; + const group = (c.scope && c.scope.group) === 'day' ? 'one fact per day' : 'one fact per dictation'; + return '<div class="muted-xs" data-kb-wisprflow-source>Wispr Flow: ' + (path ? '<code>' + escapeHtml(path) + '</code>' : 'default app location on the hive host') + ' · ' + group + '</div>'; + } + function kbOpenWisprFlowIntegration() { _kbIntegrationSelected = 'wisprflow'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } diff --git a/src/pkg/dashboard/static-src/js/layout.js b/src/pkg/dashboard/static-src/js/layout.js index b93f6f532b..ee264f9098 100644 --- a/src/pkg/dashboard/static-src/js/layout.js +++ b/src/pkg/dashboard/static-src/js/layout.js @@ -28,7 +28,7 @@ var DASHBOARD_LAYOUT_KEY='hive.dashboard.layout'; var DASHBOARD_LAYOUT_VERSION=1; var DASHBOARD_LAYOUT_REGIONS=['main']; - var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','runs-section','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','search-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; + var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','runs-section','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','search-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; var DASHBOARD_LAYOUT_GRIP_SELECTOR='[data-dashboard-grip]'; var DASHBOARD_LAYOUT_CARD_SELECTOR='[data-dashboard-section]'; var DASHBOARD_LAYOUT_ANCHOR_ID='dash-notices'; diff --git a/src/pkg/dashboard/static-src/js/linear-repos-save.js b/src/pkg/dashboard/static-src/js/linear-repos-save.js index a1224c46a2..d3fcf47919 100644 --- a/src/pkg/dashboard/static-src/js/linear-repos-save.js +++ b/src/pkg/dashboard/static-src/js/linear-repos-save.js @@ -529,7 +529,7 @@ return; } - // Requirement #3: verify the App can access the repo's org. + // Verify the App installation covers this repository. setStatus('Checking GitHub App access…', 'var(--muted)'); try { const res = await fetch('/api/config/governor/repos/check-access', { @@ -538,14 +538,11 @@ }); const d = await res.json().catch(() => ({})); if (res.status === 409 && d && d.needsInstall) { - // App not installed on the new org: surface the correct per-forge - // install URL wired to the existing pending-install/recheck plumbing. - // Clicking Install flags the heartbeat (handleGitHubAppInstallClicked) - // exactly like the App-required banner; Re-check re-probes and, on - // success, completes the add. + // Guide the operator to install the App or update repository access, + // then re-check before adding the repo. const installBtn = (d.installUrl) - ? '<a href="' + esc(d.installUrl) + '" target="_blank" rel="noopener" data-action="markGitHubAppInstallClicked" style="color:var(--accent);font-weight:600">Install/authorize the App for ' + esc(d.org || '') + ' ↗</a>' - : '<span class="text-brand">This hive\'s GitHub Enterprise App slug is not configured, so an install link cannot be built — ask your operator to install the Hive App for ' + esc(d.org || '') + '.</span>'; + ? '<a href="' + esc(d.installUrl) + '" target="_blank" rel="noopener" data-action="markGitHubAppInstallClicked" style="color:var(--accent);font-weight:600">Manage App access for ' + esc(d.org || '') + ' ↗</a>' + : '<span class="text-brand">An installation link cannot be built — ask your operator to manage the Hive App access for ' + esc(d.org || '') + '.</span>'; setStatus('⚠ ' + esc(d.error || ('The App needs access to ' + val)) + '<br>' + installBtn + '  ·  <a href="#" data-action="recheckRepoAccessThenAdd" data-arg-types="e" style="color:var(--accent);font-weight:600">Re-check & add</a>', 'var(--amber)'); return; } @@ -554,11 +551,7 @@ return; } } catch (e) { - // Network error probing access is non-fatal: the server re-validates the - // single-host rule and the default-repo rule on save, so let the add - // proceed rather than hard-block on a transient dashboard hiccup. - _appendRepoAccepted(val); - setStatus('Added (App-access check could not be reached; the server will re-validate on save).', 'var(--muted)'); + setStatus('✕ Could not check GitHub App access. Please retry.', 'var(--red)'); return; } _appendRepoAccepted(val); diff --git a/src/pkg/dashboard/static-src/js/refresh.js b/src/pkg/dashboard/static-src/js/refresh.js index e34ce97128..97242d1b2b 100644 --- a/src/pkg/dashboard/static-src/js/refresh.js +++ b/src/pkg/dashboard/static-src/js/refresh.js @@ -565,6 +565,7 @@ ocUpdateSidebarAgents(); dashboardRenderSection('debug', ['debug-section'], () => renderDebugSection(), data); dashboardRenderSection('faq', ['faq-section'], () => renderFAQ(), data); + dashboardRenderSection('inbox', ['inbox-section'], () => renderInbox(), data); initializeDashboardSectionCards(); // Restore persisted collapsed state for all collapsible sections const COLLAPSIBLE_SECTION_IDS = [ @@ -573,7 +574,7 @@ 'advisory-section', 'advisory-digest-section', 'hive-advice-section', 'acmm-reco-section', 'lifecycle-section', 'pr-throughput-section', 'overview-section', 'governor', 'token-panel', 'cost-panel', 'repos-section', 'acmm-eval-section', 'fleet-report-section', 'audit-section', 'review-queue-section', 'nous-section', 'inception-section', - 'knowledge-section', 'search-section', 'contributors-section', 'debug-section', 'logs-section', + 'knowledge-section', 'search-section', 'inbox-section', 'contributors-section', 'debug-section', 'logs-section', 'activity-monitor-section', 'agents-section', 'agent-activity-section', 'faq-section' ]; COLLAPSIBLE_SECTION_IDS.forEach(applySectionCollapse); diff --git a/src/pkg/dashboard/static-src/js/sections.js b/src/pkg/dashboard/static-src/js/sections.js index 874d7bcd42..f7b3cdb1f2 100644 --- a/src/pkg/dashboard/static-src/js/sections.js +++ b/src/pkg/dashboard/static-src/js/sections.js @@ -13,7 +13,7 @@ // marker key records whether the browser predates the defaults ('legacy', // keeps the historical all-expanded default forever) or is new ('v1'). const DASHBOARD_DEFAULT_COLLAPSED_SECTIONS = Object.freeze([ - 'pr-throughput-section', 'token-panel', 'cost-panel', 'advisory-section', 'knowledge-section', + 'pr-throughput-section', 'token-panel', 'cost-panel', 'advisory-section', 'knowledge-section', 'inbox-section', 'contributors-section', 'inception-section', 'acmm-eval-section', 'debug-section', 'audit-section', 'logs-section', 'nous-section', 'faq-section', 'runs-section', 'search-section' ]); @@ -129,6 +129,7 @@ 'inception-section': { title: '💡 Inception (spektacular)', summary: 'ready', badgeHtml: '<a class="dash-card-badge" href="https://github.com/jumppad-labs/spektacular" target="_blank" rel="noopener noreferrer" data-stop="1" title="Spektacular project">⚡ Powered by Spektacular</a>' }, 'knowledge-section': { title: '📚 Knowledge', summary: '0 facts' }, 'search-section': { title: '🔍 Search', summary: 'search' }, + 'inbox-section': { title: '📥 Inbox', summary: '0 to triage' }, 'contributors-section': { title: '👥 Contributors', summary: '0 online / 0 registered' }, 'debug-section': { title: '🩺 Diagnostics', summary: 'health' }, 'logs-section': { title: '📋 Agent Logs', summary: 'logs' }, @@ -167,6 +168,7 @@ 'inception-section': { heading: 'Inception', help: 'A step-by-step guide that turns a new project idea into a first set of files and planned work.' }, 'knowledge-section': { heading: 'Knowledge', help: 'The facts Hive has learned about your projects, where they came from, and how much Hive trusts each one.' }, 'search-section': { heading: 'Search', help: 'Find past pull requests, issues, review records and findings across your projects by keyword, with a link to each one.' }, + 'inbox-section': { heading: 'Inbox', help: 'Voice recordings and dictations waiting for you to file them as an RFC, send to dibs, or keep as knowledge.' }, 'contributors-section': { heading: 'Contributors', help: 'The people who lend their computers or AI helpers to your projects, and how far Hive trusts each of them.' }, 'debug-section': { heading: 'Diagnostics', help: 'Health checks for Hive itself, such as its connections, credentials and safety switches, to help you find out why something is not working.' }, 'logs-section': { heading: 'Agent Logs', help: 'The raw text output of each worker, for following what it is doing line by line.' }, diff --git a/src/pkg/dashboard/static-src/js/tracker-human-gate.js b/src/pkg/dashboard/static-src/js/tracker-human-gate.js index 6bab762a8a..04ba8f70c8 100644 --- a/src/pkg/dashboard/static-src/js/tracker-human-gate.js +++ b/src/pkg/dashboard/static-src/js/tracker-human-gate.js @@ -83,7 +83,11 @@ const issueDecision = typeof STAGE_TRACKER_ISSUE_NEEDS_DECISION === 'number' ? STAGE_TRACKER_ISSUE_NEEDS_DECISION : 45; // A pre-045 issue still wearing 085 is parked on a person all the same. const decisionStage = pr ? stage === prDecision : (stage === issueDecision || stage === prDecision); - const flagged = decisionStage || mods.indexOf('needs-decision') >= 0 || mods.indexOf('needs-human') >= 0 || item.needs_human === true; + // A DCO sign-off ask (dco-missing, hive-signoff-attribution) is the + // author's move, named by the waiting-on pill; needs_human alone does + // not make it a decision (hivecommons/hive#17789). + const authorDCO = /^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim()); + const flagged = decisionStage || mods.indexOf('needs-decision') >= 0 || mods.indexOf('needs-human') >= 0 || (item.needs_human === true && !authorDCO); // An unauthorized step and a structured blocker name the person owed // in their own pills (hive-bot-lacks-permission, Blocked · …); they // are not a decision anyone is asked for. diff --git a/src/pkg/dashboard/static-src/js/tracker-words.js b/src/pkg/dashboard/static-src/js/tracker-words.js index b0e95b991e..df5063fcd7 100644 --- a/src/pkg/dashboard/static-src/js/tracker-words.js +++ b/src/pkg/dashboard/static-src/js/tracker-words.js @@ -343,6 +343,8 @@ const r = stageTrackerReworkOf(item); if (!r) return ''; if (r.state === 'requested' && r.evidence === 'queued') return 'queued'; + // A DCO ask on the pushed head queues no re-review (hivecommons/hive#17789). + if (r.state === 'pushed' && /^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim())) return 'sign-off owed'; return STAGE_TRACKER_REWORK_WORDS[r.state]; } // "re-review pending" means the HIVE's reviewer owes a pass on the new @@ -360,6 +362,7 @@ const on = k => { const a = item[k + '_activity']; return a && (!a.head || !item.head || a.head === item.head) ? a.state || 'unknown' : 'unknown'; }; if (on('ci') === 'failing' || on('review') === 'changes_requested') return false; if (/conflict/i.test(String(item.reason || '')) && !item.rebase) return false; + if (/^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim())) return false; return true; } // "re-review of d88928e" — the owed pass, named by head. diff --git a/src/pkg/dashboard/static-src/manifest.txt b/src/pkg/dashboard/static-src/manifest.txt index 45731eba3a..d6c8e81100 100644 --- a/src/pkg/dashboard/static-src/manifest.txt +++ b/src/pkg/dashboard/static-src/manifest.txt @@ -29,6 +29,7 @@ css/workflows-marketplace.css css/chat.css css/nous.css css/knowledge.css +css/inbox.css css/sections-collapse.css css/design-system.css css/responsive.css @@ -56,6 +57,7 @@ html/repos.html html/acmm-approvals-audit.html html/review-queue.html html/nous-inception-knowledge-contributors.html +html/inbox.html html/debug-logs.html html/agents.html html/tracker.html @@ -119,6 +121,7 @@ js/evidence.js js/approvals.js js/nous.js js/knowledge.js +js/inbox.js js/vault-audit.js js/review-queue.js js/contributors.js diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 6e22a10350..f084cad163 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -6175,6 +6175,23 @@ #kb-gs-cancel[hidden] { display: none; } @media (max-width: 768px) { .kb-layout { grid-template-columns: 1fr; } } + /* Inbox (voice recordings / dictations triage) */ + .inbox-toolbar { flex-wrap: wrap; gap: var(--sp-3); margin-bottom: var(--sp-4); } + .inbox-sources { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); } + .inbox-toggle { display: inline-flex; align-items: center; gap: var(--sp-2); } + .inbox-list { max-height: none; } + .inbox-item { cursor: default; } + .inbox-check { display: flex; align-items: flex-start; padding-top: var(--sp-1); } + .inbox-icon { flex-shrink: 0; font-size: var(--fs-lg); line-height: 1; padding-top: var(--sp-1); } + .inbox-pills { display: flex; flex-wrap: wrap; align-items: center; gap: var(--sp-2); margin-top: var(--sp-2); } + .inbox-link { font-size: var(--fs-xs); } + .inbox-field { display: block; margin-top: var(--sp-4); } + .inbox-settings-grid { display: grid; grid-template-columns: 1fr 1fr; gap: var(--sp-4); } + .inbox-rule { display: grid; grid-template-columns: minmax(0, 1fr) minmax(0, 1.4fr) minmax(0, 1fr) minmax(0, 1.2fr) minmax(0, 1.4fr) auto; gap: var(--sp-2); align-items: center; margin-bottom: var(--sp-2); } + .kb-plaud-paste textarea { width: 100%; margin-top: var(--sp-2); font-family: var(--font-mono); font-size: var(--fs-xs); } + @media (max-width: 720px) { + .inbox-settings-grid, .inbox-rule { grid-template-columns: 1fr; } + } /* Collapsible advisory sections (#1546) */ .collapse-chevron { display: inline-block; @@ -8911,6 +8928,7 @@ <h2 class="config-title">🐝 Getting Started<span id="welcome-progress" class=" <a class="oc-nav-item" data-section="inception-section" data-action="ocNavigate" data-arg0="inception-section"><span class="oc-nav-emoji">💡</span><span class="oc-nav-text">Inception (spektacular)</span></a> <a class="oc-nav-item" data-section="knowledge-section" data-action="ocNavigate" data-arg0="knowledge-section"><span class="oc-nav-emoji">📚</span><span class="oc-nav-text">Knowledge</span><span id="knowledge-sidebar-badge" class="badge-count oc-nav-count" data-zero>0</span></a> <a class="oc-nav-item" data-section="search-section" data-action="ocNavigate" data-arg0="search-section" hidden><span class="oc-nav-emoji">🔍</span><span class="oc-nav-text">Search</span></a> + <a class="oc-nav-item" data-section="inbox-section" data-action="ocNavigate" data-arg0="inbox-section"><span class="oc-nav-emoji">📥</span><span class="oc-nav-text">Inbox</span><span id="inbox-sidebar-badge" class="badge-count oc-nav-count" data-zero hidden>0</span></a> <a class="oc-nav-item" data-section="nous-section" data-feature="experiment_lab" hidden data-action="ocNavigate" data-arg0="nous-section"><span class="oc-nav-emoji">🧪</span><span class="oc-nav-text">Experiment Lab (nous)</span></a> </div> <div class="oc-nav-group"> @@ -9379,6 +9397,12 @@ <h2 class="section-label m-0">👥 Contributors <span id="contributors-status-ba </div> + <div class="mt-6 mb-8" id="inbox-section" data-dashboard-section="inbox-section"> + <h2 class="section-label section-header-toggle mb-4" data-action="toggleSection" data-arg0="inbox-section"><button type="button" class="dashboard-grip" data-dashboard-grip data-stop="1" aria-label="Move Inbox section" aria-pressed="false">⠿</button><span class="section-chevron">▼</span> 📥 Inbox</h2> + <div class="section-body"> + <div id="inbox-panel" class="inbox-panel"></div> + </div> + </div> <div class="mt-6 mb-8" id="debug-section" data-dashboard-section="debug-section"> <h2 class="section-label section-header-toggle mb-5" data-action="toggleSection" data-arg0="debug-section"><button type="button" class="dashboard-grip" data-dashboard-grip data-stop="1" aria-label="Move Diagnostics section" aria-pressed="false">⠿</button><span class="section-chevron">▼</span> 🩺 Diagnostics</h2> <div class="section-body"> @@ -10736,7 +10760,7 @@ <h3 class="faq-group-title">Where to get help</h3> // marker key records whether the browser predates the defaults ('legacy', // keeps the historical all-expanded default forever) or is new ('v1'). const DASHBOARD_DEFAULT_COLLAPSED_SECTIONS = Object.freeze([ - 'pr-throughput-section', 'token-panel', 'cost-panel', 'advisory-section', 'knowledge-section', + 'pr-throughput-section', 'token-panel', 'cost-panel', 'advisory-section', 'knowledge-section', 'inbox-section', 'contributors-section', 'inception-section', 'acmm-eval-section', 'debug-section', 'audit-section', 'logs-section', 'nous-section', 'faq-section', 'runs-section', 'search-section' ]); @@ -10852,6 +10876,7 @@ <h3 class="faq-group-title">Where to get help</h3> 'inception-section': { title: '💡 Inception (spektacular)', summary: 'ready', badgeHtml: '<a class="dash-card-badge" href="https://github.com/jumppad-labs/spektacular" target="_blank" rel="noopener noreferrer" data-stop="1" title="Spektacular project">⚡ Powered by Spektacular</a>' }, 'knowledge-section': { title: '📚 Knowledge', summary: '0 facts' }, 'search-section': { title: '🔍 Search', summary: 'search' }, + 'inbox-section': { title: '📥 Inbox', summary: '0 to triage' }, 'contributors-section': { title: '👥 Contributors', summary: '0 online / 0 registered' }, 'debug-section': { title: '🩺 Diagnostics', summary: 'health' }, 'logs-section': { title: '📋 Agent Logs', summary: 'logs' }, @@ -10890,6 +10915,7 @@ <h3 class="faq-group-title">Where to get help</h3> 'inception-section': { heading: 'Inception', help: 'A step-by-step guide that turns a new project idea into a first set of files and planned work.' }, 'knowledge-section': { heading: 'Knowledge', help: 'The facts Hive has learned about your projects, where they came from, and how much Hive trusts each one.' }, 'search-section': { heading: 'Search', help: 'Find past pull requests, issues, review records and findings across your projects by keyword, with a link to each one.' }, + 'inbox-section': { heading: 'Inbox', help: 'Voice recordings and dictations waiting for you to file them as an RFC, send to dibs, or keep as knowledge.' }, 'contributors-section': { heading: 'Contributors', help: 'The people who lend their computers or AI helpers to your projects, and how far Hive trusts each of them.' }, 'debug-section': { heading: 'Diagnostics', help: 'Health checks for Hive itself, such as its connections, credentials and safety switches, to help you find out why something is not working.' }, 'logs-section': { heading: 'Agent Logs', help: 'The raw text output of each worker, for following what it is doing line by line.' }, @@ -12425,7 +12451,7 @@ <h3 class="faq-group-title">Where to get help</h3> var DASHBOARD_LAYOUT_KEY='hive.dashboard.layout'; var DASHBOARD_LAYOUT_VERSION=1; var DASHBOARD_LAYOUT_REGIONS=['main']; - var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','runs-section','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','search-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; + var DASHBOARD_LAYOUT_TEMPLATE={main:['overview-section','governor','runs-section','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','search-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section']}; var DASHBOARD_LAYOUT_GRIP_SELECTOR='[data-dashboard-grip]'; var DASHBOARD_LAYOUT_CARD_SELECTOR='[data-dashboard-section]'; var DASHBOARD_LAYOUT_ANCHOR_ID='dash-notices'; @@ -18636,6 +18662,8 @@ <h3 class="faq-group-title">Where to get help</h3> const r = stageTrackerReworkOf(item); if (!r) return ''; if (r.state === 'requested' && r.evidence === 'queued') return 'queued'; + // A DCO ask on the pushed head queues no re-review (hivecommons/hive#17789). + if (r.state === 'pushed' && /^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim())) return 'sign-off owed'; return STAGE_TRACKER_REWORK_WORDS[r.state]; } // "re-review pending" means the HIVE's reviewer owes a pass on the new @@ -18653,6 +18681,7 @@ <h3 class="faq-group-title">Where to get help</h3> const on = k => { const a = item[k + '_activity']; return a && (!a.head || !item.head || a.head === item.head) ? a.state || 'unknown' : 'unknown'; }; if (on('ci') === 'failing' || on('review') === 'changes_requested') return false; if (/conflict/i.test(String(item.reason || '')) && !item.rebase) return false; + if (/^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim())) return false; return true; } // "re-review of d88928e" — the owed pass, named by head. @@ -27489,7 +27518,11 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> const issueDecision = typeof STAGE_TRACKER_ISSUE_NEEDS_DECISION === 'number' ? STAGE_TRACKER_ISSUE_NEEDS_DECISION : 45; // A pre-045 issue still wearing 085 is parked on a person all the same. const decisionStage = pr ? stage === prDecision : (stage === issueDecision || stage === prDecision); - const flagged = decisionStage || mods.indexOf('needs-decision') >= 0 || mods.indexOf('needs-human') >= 0 || item.needs_human === true; + // A DCO sign-off ask (dco-missing, hive-signoff-attribution) is the + // author's move, named by the waiting-on pill; needs_human alone does + // not make it a decision (hivecommons/hive#17789). + const authorDCO = /^(dco-missing|hive-signoff-attribution)$/.test(String(item.reason || '').trim()); + const flagged = decisionStage || mods.indexOf('needs-decision') >= 0 || mods.indexOf('needs-human') >= 0 || (item.needs_human === true && !authorDCO); // An unauthorized step and a structured blocker name the person owed // in their own pills (hive-bot-lacks-permission, Blocked · …); they // are not a decision anyone is asked for. @@ -30992,11 +31025,15 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> by the Go server under the same /terminal prefix so openTerminal()'s handoff code and assertion cookie apply unchanged. openshell agents (#17645) run each kick in a gateway-owned sandbox and use the same - viewer, which streams the sandbox's log instead. */ + viewer, which streams the sandbox's log instead; substrate agents + (#17777) run each kick in a Substrate actor and the viewer streams the + kick's output from the actor's bridge. */ const JOB_SANDBOX_RUNTIME = 'job'; const OPENSHELL_SANDBOX_RUNTIME = 'openshell'; + const SUBSTRATE_SANDBOX_RUNTIME = 'substrate'; function sandboxRuntimeWithoutTmux(sandboxRuntime) { - return sandboxRuntime === JOB_SANDBOX_RUNTIME || sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME; + return sandboxRuntime === JOB_SANDBOX_RUNTIME || sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME + || sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME; } function agentUsesJobTerminal(a) { return !!a && sandboxRuntimeWithoutTmux(a.sandboxRuntime); @@ -31012,6 +31049,9 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> } function terminalLinkTitle(a, tmuxTitle) { if (!agentUsesJobTerminal(a)) return tmuxTitle; + if (a.sandboxRuntime === SUBSTRATE_SANDBOX_RUNTIME) { + return 'Stream this agent\'s kick output and Substrate actor state (read-only: it runs in a Substrate actor, not a tmux session)'; + } return a.sandboxRuntime === OPENSHELL_SANDBOX_RUNTIME ? 'Stream the log of this agent\'s OpenShell sandbox (read-only: it runs in a gateway-owned sandbox, not a tmux session)' : 'Stream the log of this agent\'s sandbox Job pod (read-only: it runs as a Kubernetes Job, not a tmux session)'; @@ -31752,6 +31792,12 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> tmux session or in-pod process to be "down". Their health comes from the server-side sandboxRun verdict (last run + cadence), never from state !== 'running'. */ + /* Preflight findings for this agent's sandbox runtime (openshell or + substrate), computed at boot/reload and carried as sandbox_warnings. */ + function agentSandboxWarningsHtml(a) { + const ws = a && Array.isArray(a.sandbox_warnings) ? a.sandbox_warnings : []; + return ws.map(w => `<div class="agent-error" title="${esc(w)}">⚠ ${esc(w)}</div>`).join(''); + } function agentSandboxRun(a) { return (a && a.sandboxed && a.sandboxRun && a.sandboxRun.health) ? a.sandboxRun : null; } @@ -31991,6 +32037,7 @@ <h2 class="config-title" id="activity-monitor-labels-title">All labels</h2> const prefix = retrying ? 'retrying: ' : ''; return `<div class="agent-error" title="${esc(a.lastError)}">⚠ ${esc(prefix)}${esc(a.lastError)}</div>`; })()} + ${agentSandboxWarningsHtml(a)} <div class="agent-card-details" data-agent-resize-y data-agent="${esc(a.name)}"${layoutHeightAttr}> <div class="agent-field"><span class="label" title="${isInferenceBackendName(a.cli) ? 'Inference backend (self-hosted model server)' : 'The CLI backend running this agent (e.g. claude, copilot). Pin to prevent governor from switching it.'}">${isInferenceBackendName(a.cli) ? 'method' : 'cli'}</span><span class="value">${cliChip(a.cli, a.pinnedBoth || a.pinnedCli, a.name)}${(a.pinnedBoth || a.pinnedCli) ? '' : ` <button class="hv-btn pin-toggle btn-icon btn-sm" data-action="togglePin" data-agent="${esc(a.name)}" data-pin-type="cli" data-unpin="false" title="Pin CLI — lock current backend">📌</button>`}</span></div> <div class="agent-field"><span class="label" title="The LLM model powering this agent. Governor may change it based on budget or mode. Pin to lock.">model</span><span class="value">${modelChip(a.model, a.govReason, a.pinnedBoth || a.pinnedModel, a.name)}${(a.pinnedBoth || a.pinnedModel) ? '' : ` <button class="hv-btn pin-toggle btn-icon btn-sm" data-action="togglePin" data-agent="${esc(a.name)}" data-pin-type="model" data-unpin="false" title="Pin model — lock current model">📌</button>`}</span></div> @@ -48194,6 +48241,18 @@ <h2 style="font-size:1rem;margin:var(--sp-0) var(--sp-0) var(--sp-0) var(--sp-0) html += '<button class="hv-btn btn-primary nous-btn btn-blue" data-action="kbImportDocFromModal">Import</button>'; html += '</div></div></details>'; + html += '<details class="kb-sources-add"><summary class="summary-label">🎙️ Plaud voice notes</summary>'; + html += '<div class="stack kb-sources-form">'; + html += '<div class="muted-xs">Make your Plaud Note recordings (Plaud summary + transcript) searchable knowledge via the free official Plaud MCP server. $0 beyond your Plaud plan; sign-in is Plaud OAuth in your browser.</div>'; + html += '<div><button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbOpenPlaudIntegration">Configure in Integrations</button></div>'; + html += '</div></details>'; + + html += '<details class="kb-sources-add"><summary class="summary-label">🗣️ Wispr Flow dictation</summary>'; + html += '<div class="stack kb-sources-form">'; + html += '<div class="muted-xs">Dictate straight into any field here — put the cursor in <strong>+ New Fact</strong> and hold the Flow key; no connector needed. To also pull your past dictations in, the Wispr Flow connector reads the app\'s local history (flow.sqlite or an export folder). $0 beyond your Wispr Flow plan.</div>'; + html += '<div><button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbOpenWisprFlowIntegration">Configure in Integrations</button></div>'; + html += '</div></details>'; + html += '<details open class="kb-sources-add"><summary class="summary-label">📚 Import from Context7</summary>'; html += '<div class="stack kb-sources-form">'; html += '<div class="muted-xs">Search Context7 for library documentation (e.g. "vllm", "next.js", "kubernetes").</div>'; @@ -48675,9 +48734,12 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> html += '<div class="kb-integration-connector-title"><code>' + escapeHtml(c.name) + '</code> <span class="kc-pill" data-status="' + escapeHtml(status) + '">' + escapeHtml(status) + '</span></div>'; html += '<div class="muted-xs">' + escapeHtml(c.layer || 'project') + ' · ' + escapeHtml(c.interval || '15m') + ' · last sync ' + escapeHtml(kcAgo(r.last_sync)) + ' · pages/facts ' + (r.pages || 0) + '/' + (r.facts || 0) + escapeHtml(publish) + '</div>'; if (r.last_error) html += '<div class="kc-error" title="' + escapeHtml(r.last_error) + '">' + escapeHtml(r.last_error.length > 140 ? r.last_error.slice(0, 140) + '…' : r.last_error) + '</div>'; + if (type === 'plaud') html += kbPlaudAuthLine(c.name) + kbPlaudPasteHtml(c.name); + if (type === 'wisprflow') html += kbWisprFlowSourceLine(c); html += '</div>'; if (owner) { html += '<div class="kb-integration-actions">'; + if (type === 'plaud') html += kbPlaudAuthButtons(c.name); html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationSync" data-arg0="' + escapeHtml(c.name) + '"' + (status === 'syncing' ? ' disabled' : '') + '>Sync now</button>'; html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationEdit" data-arg0="' + escapeHtml(c.name) + '">Edit</button>'; html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationToggle" data-arg0="' + escapeHtml(c.name) + '">' + (c.enabled ? 'Disable' : 'Enable') + '</button>'; @@ -48693,7 +48755,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> const connectors = (_kcState.connectors || []).filter(c => c.type === type); if (!_kbIntegrationEditName || !connectors.some(c => c.name === _kbIntegrationEditName)) _kbIntegrationEditName = connectors[0]?.name || ''; const edited = connectors.find(c => c.name === _kbIntegrationEditName) || { name: '', type, enabled: true, layer: 'project', interval: '', scope: {}, auth: {} }; - let html = '<div class="card-inset mb-5"><div class="row-between"><div><div class="title-sm kb-integration-title">' + kbIntegrationIcon(type) + escapeHtml(info.label) + '</div><p class="muted-note m-0">' + escapeHtml(info.description) + '</p></div><a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Docs ↗</a></div></div>'; + let html = '<div class="card-inset mb-5"><div class="row-between"><div><div class="title-sm kb-integration-title">' + kbIntegrationIcon(type) + escapeHtml(info.label) + '</div><p class="muted-note m-0">' + escapeHtml(info.description) + '</p></div><span class="row">' + (type === 'plaud' ? '<a href="/docs/inbox-voice-sources.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Inbox quickstart ↗</a>' : '') + '<a href="/docs/knowledge-connectors.md" target="_blank" rel="noopener" class="hv-btn btn-secondary btn-sm">Docs ↗</a></span></div></div>'; html += '<div class="row-between mb-4"><div class="section-label m-0">Configured entries</div>' + (owner ? '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbIntegrationNew" data-arg0="' + escapeHtml(type) + '">Add ' + escapeHtml(info.label) + '</button>' : '') + '</div>'; html += connectors.length ? connectors.map(c => kbConnectorRow(type, c, owner)).join('') : '<div class="empty-dashed mb-4">No ' + escapeHtml(info.label) + ' connector configured yet.</div>'; html += owner ? kbConnectorFormHTML(type, edited, !!edited.name) : '<p class="muted-note">Only the hive owner can edit connector configuration.</p>'; @@ -48714,6 +48776,8 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> html += '<label class="muted-note">Credential reference<input class="kb-search-box mt-2" type="text" data-kb-kc-auth-value value="' + escapeHtml(authVal) + '" placeholder="TOKEN_ENV or /run/secrets/token" autocomplete="off"></label></div>'; html += '<div class="kb-integration-scope-grid">' + kcScopeFieldsHTML(type, c.scope || {}) + '</div>'; if (type === 'context7') html += kbContext7FormExtras(c); + if (type === 'wisprflow') html += '<p class="muted-xs" data-kb-wisprflow-note>Leave the credential source as None: the connector reads Wispr Flow\'s local history read-only with the sqlite3 CLI. Leave the path empty for the default app location on the hive host (macOS: ~/Library/Application Support/Wispr Flow/flow.sqlite; Windows: %APPDATA%\\Wispr Flow\\flow.sqlite), or point it at a copy of flow.sqlite or a folder of exported .json/.ndjson/.csv/.md files. New dictations need no connector at all — dictate into + New Fact directly.</p>'; + if (type === 'plaud') html += '<p class="muted-xs" data-kb-plaud-note>Leave the credential source as None: Plaud signs in with its own OAuth flow (Connect on the entry above) and keeps the token in the MCP server\'s own store. Requires Node.js ≥ 20 on the hive host and costs nothing beyond your Plaud plan.</p>'; html += '<label class="muted-note kb-integration-enabled"><input type="checkbox" data-kb-kc-enabled' + (c.enabled ? ' checked' : '') + '> Enabled</label>'; html += '<div class="hive-dialog-error" role="alert" data-kb-kc-msg></div>'; html += '<div class="row-end kb-integration-form-actions"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbIntegrationValidate">Validate</button><button type="button" class="hv-btn btn-primary btn-sm" data-action="kbIntegrationSave">Save</button></div></div>'; @@ -48797,6 +48861,442 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> await kcSyncNow(name); setTimeout(() => kbLoadIntegrationsModal(_kbIntegrationSelected), 1500); } + + // ── Plaud sign-in (OAuth through the Plaud MCP server) ── + // Cached per connector name; loaded lazily when the Plaud card renders. + const _kbPlaudAuth = {}; + function kbPlaudAuthLine(name) { + const a = _kbPlaudAuth[name]; + if (!a) { + setTimeout(() => kbPlaudAuthLoad(name, false), 0); + return '<div class="muted-xs" data-kb-plaud-auth="' + escapeHtml(name) + '">Plaud: checking sign-in…</div>'; + } + let text; + if (a.error) text = 'Plaud: ' + a.error; + else if (a.pending) text = 'Plaud: waiting for authorization' + (a.auth_url ? ' — <a href="' + escapeHtml(a.auth_url) + '" target="_blank" rel="noopener">open the Plaud authorization page ↗</a>' : ' (URL pending…)'); + else if (a.connected) text = 'Plaud: connected' + (a.account ? ' as <strong>' + escapeHtml(a.account) + '</strong>' : ''); + else text = 'Plaud: not connected' + (a.message ? ' — ' + escapeHtml(a.message) : ''); + return '<div class="muted-xs" data-kb-plaud-auth="' + escapeHtml(name) + '" data-connected="' + (a.connected ? '1' : '0') + '">' + text + '</div>'; + } + function kbPlaudAuthButtons(name) { + const a = _kbPlaudAuth[name] || {}; + const n = escapeHtml(name); + let html = ''; + if (a.connected) html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudDisconnect" data-arg0="' + n + '">Disconnect</button>'; + else html += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="kbPlaudConnect" data-arg0="' + n + '"' + (a.pending ? ' disabled' : '') + '>Connect Plaud</button>'; + html += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudRefreshAuth" data-arg0="' + n + '" title="Re-check sign-in with Plaud">Check</button>'; + return html; + } + async function kbPlaudAuthLoad(name, refresh) { + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth' + (refresh ? '?refresh=1' : '')); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + } catch (e) { + _kbPlaudAuth[name] = { error: e.message }; + } + if (document.querySelector('[data-kb-plaud-auth="' + CSS.escape(name) + '"]')) kbRenderIntegrationsModal(); + } + async function kbPlaudRefreshAuth(name) { + showToast('Checking Plaud sign-in…', 'info'); + await kbPlaudAuthLoad(name, true); + } + async function kbPlaudConnect(name) { + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/connect', { method: 'POST' }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + if (data.connected) showToast('Plaud connected' + (data.account ? ' as ' + data.account : ''), 'success'); + else if (data.auth_url) { showToast('Open the Plaud authorization link shown on the card, then click Check.', 'info'); kbPlaudPollAuth(name, 0); } + else showToast('Plaud login started; the authorization URL will appear on the card shortly.', 'info'); + if (!data.connected && !data.auth_url) kbPlaudPollAuth(name, 0); + } catch (e) { + showToast('Plaud connect failed: ' + e.message, 'error'); + } + } + // Poll the cached state while a login is pending (URL arrival, completion). + function kbPlaudPollAuth(name, attempt) { + if (attempt > 60) return; + setTimeout(async () => { + await kbPlaudAuthLoad(name, false); + const a = _kbPlaudAuth[name] || {}; + if (a.connected) { showToast('Plaud connected' + (a.account ? ' as ' + a.account : ''), 'success'); return; } + if (a.pending) kbPlaudPollAuth(name, attempt + 1); + }, 5000); + } + async function kbPlaudDisconnect(name) { + if (!(await hiveConfirm('Sign out of Plaud? Synced recordings stay in the vault until you remove the connector.'))) return; + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/disconnect', { method: 'POST' }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + showToast('Plaud disconnected', 'success'); + } catch (e) { + showToast('Plaud disconnect failed: ' + e.message, 'error'); + } + } + function kbOpenPlaudIntegration() { _kbIntegrationSelected = 'plaud'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } + + // ── Wispr Flow (local dictation history, no sign-in) ── + function kbWisprFlowSourceLine(c) { + const path = (c.scope && c.scope.path) || ''; + const group = (c.scope && c.scope.group) === 'day' ? 'one fact per day' : 'one fact per dictation'; + return '<div class="muted-xs" data-kb-wisprflow-source>Wispr Flow: ' + (path ? '<code>' + escapeHtml(path) + '</code>' : 'default app location on the hive host') + ' · ' + group + '</div>'; + } + function kbOpenWisprFlowIntegration() { _kbIntegrationSelected = 'wisprflow'; _kbIntegrationEditName = ''; kbOpenIntegrations(); } + // ── Inbox: source-agnostic triage of voice recordings and dictations ── + // Items come from knowledge connectors (plaud, wisprflow, …); each one is + // already a Knowledge fact. Routing them to spek (RFC + child issues) or + // dibs happens inside the hive via POST /api/inbox/route. + const INBOX_POLL_MS = 60000; + const INBOX_BUSY_POLL_MS = 5000; + const INBOX_SOURCE_ICONS = { plaud: '🎙️', wisprflow: '🗣️', 'file-drop': '📎', webhook: '🔗' }; + const INBOX_DEST_LABELS = { spek_rfc: 'RFC', dibs: 'dibs' }; + let _inboxCache = null; + let _inboxSelected = new Set(); + let _inboxShowProcessed = false; + let _inboxBusyTimer = null; + + function inboxSourceIcon(src) { return INBOX_SOURCE_ICONS[src] || '📥'; } + function inboxItemKey(it) { return it.source + '/' + it.id; } + function inboxFmtDuration(ms) { + const s = Math.round((Number(ms) || 0) / 1000); + if (s <= 0) return ''; + const m = Math.floor(s / 60); + return m >= 60 ? Math.floor(m / 60) + 'h ' + (m % 60) + 'm' : m + 'm ' + (s % 60) + 's'; + } + function inboxRouteBusy(items) { + return (items || []).some(it => Object.values(it.routes || {}).some(r => r && (r.status === 'queued' || r.status === 'running'))); + } + + async function fetchInbox() { + try { + const r = await fetch('/api/inbox'); + if (!r.ok) { + // Never leave "Loading inbox…" up: a 404/500 becomes a visible error. + const body = await r.json().catch(() => ({})); + _inboxCache = { items: [], sources: [], settings: {}, error: body.error || ('HTTP ' + r.status) }; + renderInbox(); + return; + } + _inboxCache = await r.json(); + renderInbox(); + clearTimeout(_inboxBusyTimer); + if (inboxRouteBusy(_inboxCache.items)) _inboxBusyTimer = setTimeout(fetchInbox, INBOX_BUSY_POLL_MS); + } catch (e) { /* transient — leave last render in place */ } + } + dashboardScheduleWork('inbox', fetchInbox).catch(console.error); + dashboardSetInterval('inbox', fetchInbox, INBOX_POLL_MS); + + function inboxStatePill(dest, st) { + const label = INBOX_DEST_LABELS[dest] || dest; + if (!st) return ''; + const status = st.status === 'filed' || st.status === 'delegated' ? 'ok' : (st.status === 'failed' ? 'error' : 'syncing'); + const text = st.status === 'filed' ? label + ' filed' : st.status === 'delegated' ? label + ' handed to agent' : label + ' ' + st.status; + const title = st.error ? escapeHtml(st.error) : (st.repo ? escapeHtml(st.repo) : ''); + let html = '<span class="kc-pill" data-status="' + status + '" title="' + title + '">' + escapeHtml(text) + '</span>'; + if (st.url) html += ' <a class="inbox-link" href="' + escapeHtml(st.url) + '" target="_blank" rel="noopener">open ↗</a>'; + if (Array.isArray(st.children) && st.children.length) html += ' <span class="muted-xs">' + st.children.length + ' child issue' + (st.children.length === 1 ? '' : 's') + '</span>'; + return html; + } + + function inboxItemHtml(it) { + const key = inboxItemKey(it); + const checked = _inboxSelected.has(key) ? ' checked' : ''; + const when = it.recorded_at && !String(it.recorded_at).startsWith('0001') ? relativeTimeShort(it.recorded_at) : ''; + const dur = inboxFmtDuration(it.duration_ms); + const meta = [escapeHtml(it.source), when, dur].filter(Boolean).join(' · '); + const routes = it.routes || {}; + let pills = ''; + for (const d of Object.keys(routes)) pills += inboxStatePill(d, routes[d]) + ' '; + if (it.dismissed) pills += '<span class="kc-pill" title="Kept as a Knowledge fact only">knowledge only</span>'; + const owner = viewerIsOwner(); + const src = escapeHtml(it.source), id = escapeHtml(it.id); + let actions = ''; + if (owner) { + const retry = (d) => routes[d] && routes[d].status === 'failed' ? 'Retry ' : '→ '; + actions += '<button type="button" class="hv-btn btn-primary btn-sm" data-action="inboxOpenRoute" data-arg0="' + src + '" data-arg1="' + id + '" data-arg2="spek_rfc">' + retry('spek_rfc') + 'RFC…</button>'; + actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenRoute" data-arg0="' + src + '" data-arg1="' + id + '" data-arg2="dibs">' + retry('dibs') + 'dibs…</button>'; + if (!it.dismissed) actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismiss" data-arg0="' + src + '" data-arg1="' + id + '" title="Keep it as a Knowledge fact; no RFC, no dibs idea">Knowledge only</button>'; + else actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxUndismiss" data-arg0="' + src + '" data-arg1="' + id + '">Back to inbox</button>'; + } + if (it.fact_slug) actions += '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenFact" data-arg0="' + escapeHtml(it.fact_slug) + '" data-arg1="' + escapeHtml(it.layer || '') + '" title="This item is already a Knowledge fact">📚 Fact</button>'; + return '<div class="kb-fact-item inbox-item" data-inbox-key="' + escapeHtml(key) + '">' + + (owner ? '<label class="inbox-check"><input type="checkbox" data-change-action="inboxToggleSelect" data-arg0="' + src + '" data-arg1="' + id + '"' + checked + ' aria-label="Select ' + escapeHtml(it.title) + '"></label>' : '') + + '<div class="inbox-icon" title="' + src + '">' + inboxSourceIcon(it.source) + '</div>' + + '<div class="kb-fact-body"><div class="kb-fact-title">' + escapeHtml(it.title || it.id) + '</div>' + + '<div class="muted-xs">' + meta + '</div>' + + (it.excerpt ? '<div class="kb-fact-snippet">' + escapeHtml(it.excerpt) + '</div>' : '') + + (pills ? '<div class="inbox-pills">' + pills + '</div>' : '') + + '<div class="kc-actions mt-2">' + actions + '</div></div></div>'; + } + + function renderInbox() { + refreshSectionCardShell('inbox-section'); + const panel = document.getElementById('inbox-panel'); + const badge = document.getElementById('inbox-sidebar-badge'); + const data = _inboxCache; + if (!panel) return; + if (!data) { panel.innerHTML = '<div class="muted-note">Loading inbox…</div>'; return; } + const items = data.items || []; + const unprocessed = Number(data.unprocessed) || 0; + if (badge) setSidebarCountBadge(badge, unprocessed, unprocessed + ' unprocessed'); + setSectionSummary('inbox-section', unprocessed + ' to triage · ' + items.length + ' total'); + const owner = viewerIsOwner(); + const sources = (data.sources || []).map(s => { + const status = s.connected ? 'ok' : (s.configured ? 'error' : ''); + const label = inboxSourceIcon(s.type) + ' ' + escapeHtml(s.type) + (s.connected ? (s.account ? ': ' + escapeHtml(s.account) : ': connected') : s.configured ? ': not connected' : ': not configured'); + return '<span class="kc-pill" data-status="' + status + '">' + label + '</span>'; + }).join(' '); + const settings = data.settings || {}; + const rules = Array.isArray(settings.rules) ? settings.rules.length : 0; + let html = '<div class="row-between inbox-toolbar">' + + '<div class="inbox-sources">' + (sources || '<span class="muted-xs">No voice sources configured.</span>') + + ' <button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbOpenIntegrations">Connect sources</button></div>' + + '<div class="kc-actions">' + + '<label class="muted-xs inbox-toggle"><input type="checkbox" data-change-action="inboxToggleProcessed"' + (_inboxShowProcessed ? ' checked' : '') + '> show processed</label>' + + (owner ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxOpenSettings" title="Auto-routing rules, default RFC repository, dibs URL">⚙ Auto-routing (' + rules + ')</button>' : '') + + '</div></div>'; + if (data.error) html += '<div class="kc-error muted-note">' + escapeHtml(data.error) + '</div>'; + if (!settings.spek_ready && owner) html += '<p class="muted-xs">RFC filing needs a <code>spektacular</code> binary on the hive host (<code>inbox.spek.binary</code>) or at least one enabled agent to hand the job to (<code>inbox.spek.agent</code> overrides the default).</p>'; + const shown = items.filter(it => _inboxShowProcessed || !it.processed); + if (owner && shown.length) { + const sel = _inboxSelected.size; + html += '<div class="kc-actions mb-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxSelectAll">' + (sel === shown.length ? 'Clear selection' : 'Select all') + '</button>' + + (sel ? '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxDismissSelected">Knowledge only (' + sel + ')</button>' : '') + '</div>'; + } + if (!shown.length) html += '<div class="muted-note">' + (items.length ? 'Everything is triaged. Toggle “show processed” to see filed and dismissed items.' : 'Nothing yet. Connect Plaud or another voice source under Knowledge → Integrations; new recordings land here after the next sync (a newly added connector starts on the next hive restart).') + '</div>'; + else html += '<div class="kb-fact-list inbox-list">' + shown.map(inboxItemHtml).join('') + '</div>'; + panel.innerHTML = html; + } + + function inboxToggleProcessed() { _inboxShowProcessed = !_inboxShowProcessed; renderInbox(); } + function inboxToggleSelect(src, id) { + const key = src + '/' + id; + if (_inboxSelected.has(key)) _inboxSelected.delete(key); else _inboxSelected.add(key); + renderInbox(); + } + function inboxSelectAll() { + const shown = ((_inboxCache || {}).items || []).filter(it => _inboxShowProcessed || !it.processed); + if (_inboxSelected.size === shown.length) _inboxSelected.clear(); + else _inboxSelected = new Set(shown.map(inboxItemKey)); + renderInbox(); + } + function inboxFind(src, id) { return ((_inboxCache || {}).items || []).find(it => it.source === src && it.id === id); } + function inboxOpenFact(slug, layer) { + ocNavigate('knowledge-section'); + kbReadFact(slug, layer || undefined); + } + + async function inboxPostDismiss(items, undo) { + const r = await fetch('/api/inbox/dismiss', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ items, undo: !!undo }) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + return data; + } + async function inboxDismiss(src, id) { + try { await inboxPostDismiss([{ source: src, item_id: id }], false); showToast('Kept as a Knowledge fact only', 'success'); await fetchInbox(); } + catch (e) { showToast('Dismiss failed: ' + e.message, 'error'); } + } + async function inboxUndismiss(src, id) { + try { await inboxPostDismiss([{ source: src, item_id: id }], true); await fetchInbox(); } + catch (e) { showToast('Undo failed: ' + e.message, 'error'); } + } + async function inboxDismissSelected() { + const items = Array.from(_inboxSelected).map(k => { const i = k.indexOf('/'); return { source: k.slice(0, i), item_id: k.slice(i + 1) }; }); + if (!items.length) return; + if (!(await hiveConfirm('Mark ' + items.length + ' item' + (items.length === 1 ? '' : 's') + ' as Knowledge only? They stay searchable as facts; no RFC or dibs idea is created.', 'Knowledge only'))) return; + try { await inboxPostDismiss(items, false); _inboxSelected.clear(); await fetchInbox(); } + catch (e) { showToast('Dismiss failed: ' + e.message, 'error'); } + } + + // Route dialog: destination-specific confirm with a repo picker for RFCs. + function inboxOpenRoute(src, id, dest) { + const it = inboxFind(src, id); + if (!it) return; + const settings = (_inboxCache || {}).settings || {}; + const repos = (_inboxCache || {}).repos || []; + const def = settings.spek_repo || repos[0] || ''; + let body = '<p class="muted-note m-0"><strong>' + escapeHtml(it.title || it.id) + '</strong></p>'; + if (dest === 'spek_rfc') { + const opts = repos.map(r => '<option value="' + escapeHtml(r) + '"' + (r === def ? ' selected' : '') + '>' + escapeHtml(r) + '</option>').join(''); + body += '<p class="muted-note">A hive-run <code>spektacular voice</code> turns the transcript into a spec, then files one RFC issue plus child issues on the chosen repository. The children are picked up by the hive like any other issue.</p>' + + '<label class="muted-note inbox-field">Repository<select class="field-input field-input-block mt-2" id="inbox-route-repo">' + opts + '</select></label>' + + (repos.length ? '' : '<p class="kc-error muted-xs">No repositories configured on this hive.</p>'); + } else { + body += '<p class="muted-note">Posts the title, summary and transcript to dibs as a private draft idea' + (settings.dibs_url ? ' at <code>' + escapeHtml(settings.dibs_url) + '</code>' : '') + '. Provenance links back to <code>' + escapeHtml(src) + '://recordings/' + escapeHtml(id) + '</code>.</p>' + + (!settings.dibs_url ? '<p class="kc-error muted-xs">dibs is not configured — set the URL under ⚙ Auto-routing and <code>inbox.dibs.token_env</code> in the hive config.</p>' : !settings.dibs_token_set ? '<p class="kc-error muted-xs">dibs token missing — set the environment variable named by <code>inbox.dibs.token_env</code>.</p>' : ''); + } + body += '<div class="hive-dialog-error" role="alert" id="inbox-route-error"></div>' + + '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxConfirmRoute" data-arg0="' + escapeHtml(src) + '" data-arg1="' + escapeHtml(id) + '" data-arg2="' + escapeHtml(dest) + '">' + (dest === 'spek_rfc' ? 'File RFC' : 'Send to dibs') + '</button></div>'; + kbShowModal(dest === 'spek_rfc' ? '→ RFC (spektacular)' : '→ dibs', body, { width: '520px' }); + } + async function inboxConfirmRoute(src, id, dest) { + const repoEl = document.getElementById('inbox-route-repo'); + const repo = repoEl ? repoEl.value : ''; + const errEl = document.getElementById('inbox-route-error'); + try { + const r = await fetch('/api/inbox/route', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ source: src, item_id: id, destination: dest, repo }) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + kbCloseModal(); + showToast((INBOX_DEST_LABELS[dest] || dest) + ' queued — the pill updates when it finishes.', 'success'); + await fetchInbox(); + } catch (e) { + if (errEl) errEl.textContent = e.message; else showToast('Route failed: ' + e.message, 'error'); + } + } + + // Settings drawer: default RFC repo, dibs URL, and ordered auto-routing rules. + let _inboxDraftRules = []; + function inboxRuleRowHtml(rule, i) { + const sources = ((_inboxCache || {}).settings || {}).sources || []; + const srcOpts = ['<option value=""' + (!rule.source ? ' selected' : '') + '>any source</option>'].concat(sources.map(s => '<option value="' + escapeHtml(s) + '"' + (rule.source === s ? ' selected' : '') + '>' + escapeHtml(s) + '</option>')).join(''); + const destOpts = [['spek_rfc', '→ RFC'], ['dibs', '→ dibs'], ['dismiss', 'Knowledge only']].map(([v, l]) => '<option value="' + v + '"' + (rule.destination === v ? ' selected' : '') + '>' + l + '</option>').join(''); + return '<div class="inbox-rule" data-rule-index="' + i + '">' + + '<select class="field-input" data-rule-field="source" aria-label="Source">' + srcOpts + '</select>' + + '<input class="kb-search-box" data-rule-field="title_prefix" placeholder="title prefix (RFC:)" value="' + escapeHtml(rule.title_prefix || '') + '" aria-label="Title prefix">' + + '<input class="kb-search-box" data-rule-field="tag" placeholder="or tag" value="' + escapeHtml(rule.tag || '') + '" aria-label="Tag">' + + '<select class="field-input" data-rule-field="destination" aria-label="Destination">' + destOpts + '</select>' + + '<input class="kb-search-box" data-rule-field="repo" placeholder="owner/repo (RFC only)" value="' + escapeHtml(rule.repo || '') + '" aria-label="Repository">' + + '<button type="button" class="hv-btn btn-icon btn-sm" data-action="inboxRemoveRule" data-arg0="' + i + '" aria-label="Remove rule">✕</button></div>'; + } + // The same form renders in the Inbox page modal and inline under + // Settings → Inbox, and both can sit in the DOM at once (the settings + // tab stays mounted; the closed modal keeps its last body). Every read + // and write therefore goes through the active form root: the modal + // while it is open, otherwise the inline section. + function inboxSettingsRoot() { + const overlay = document.getElementById('kb-modal-overlay'); + if (overlay && overlay.style.display !== 'none') { + const modal = overlay.querySelector('.inbox-settings-form'); + if (modal) return modal; + } + return document.querySelector('#inbox-settings-inline .inbox-settings-form'); + } + function inboxSettingsField(root, name) { + return root ? root.querySelector('[data-inbox-field="' + name + '"]') : null; + } + function inboxReadRules(root) { + root = root || inboxSettingsRoot(); + if (!root) return []; + return Array.from(root.querySelectorAll('.inbox-settings-rules .inbox-rule')).map(row => { + const v = f => { const el = row.querySelector('[data-rule-field="' + f + '"]'); return el ? el.value.trim() : ''; }; + return { source: v('source'), title_prefix: v('title_prefix'), tag: v('tag'), destination: v('destination'), repo: v('repo') }; + }); + } + function inboxRenderRules(root) { + root = root || inboxSettingsRoot(); + const host = root ? root.querySelector('.inbox-settings-rules') : null; + if (!host) return; + host.innerHTML = _inboxDraftRules.length ? _inboxDraftRules.map(inboxRuleRowHtml).join('') : '<p class="muted-xs">No rules — every item waits here for manual triage.</p>'; + } + function inboxSettingsFormHtml(settings, inline) { + const buttons = inline + ? '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>' + : '<div class="hive-dialog-buttons"><button type="button" class="hv-btn btn-secondary hive-dialog-btn" data-action="kbCloseModal">Cancel</button>' + + '<button type="button" class="hv-btn btn-primary hive-dialog-btn primary" data-action="inboxSaveSettings">Save</button></div>'; + return '<div class="inbox-settings-form" data-inbox-form="' + (inline ? 'inline' : 'modal') + '"><p class="muted-note m-0">New items are checked against these rules in order after every connector sync; the first match routes the item automatically. Items that arrived before a rule existed are never re-routed.</p>' + + '<div class="inbox-settings-grid">' + + '<label class="muted-note inbox-field">Default RFC repository<input class="kb-search-box mt-2" type="text" data-inbox-field="spek-repo" placeholder="owner/repo" autocomplete="off" value="' + escapeHtml(settings.spek_repo || '') + '"></label>' + + '<label class="muted-note inbox-field">dibs URL<input class="kb-search-box mt-2" type="text" data-inbox-field="dibs-url" placeholder="https://dibs.example.org" autocomplete="off" value="' + escapeHtml(settings.dibs_url || '') + '"></label></div>' + + '<p class="muted-xs">The dibs token is read from the environment variable named by <code>inbox.dibs.token_env</code>' + (settings.dibs_token_set ? ' (set)' : ' (not set)') + '.</p>' + + '<div class="row-between mt-4 mb-2"><div class="section-label m-0">Rules</div><button type="button" class="hv-btn btn-secondary btn-sm" data-action="inboxAddRule">Add rule</button></div>' + + '<div class="inbox-settings-rules"></div>' + + '<div class="hive-dialog-error" role="alert" data-inbox-field="error"></div>' + buttons + '</div>'; + } + function inboxOpenSettings() { + const settings = (_inboxCache || {}).settings || {}; + _inboxDraftRules = (settings.rules || []).map(r => Object.assign({}, r)); + kbShowModal('⚙ Inbox auto-routing', inboxSettingsFormHtml(settings, false), { width: '720px' }); + inboxRenderRules(); + } + // Settings → Inbox: the same form inline, loaded from /api/inbox/settings. + function renderGovInbox() { + return '<div class="config-section" data-config-section="Inbox"><h3>Inbox</h3>' + + '<p class="text-muted fs-base">Default RFC repository, dibs URL and auto-routing rules for voice recordings. ' + yamlHint('inbox.spek.repo') + ' ' + yamlHint('inbox.dibs.url') + ' ' + yamlHint('inbox.rules') + '</p>' + + '<div id="inbox-settings-inline"><p class="muted-xs">Loading…</p></div></div>'; + } + async function loadInboxSettingsInline() { + const host = document.getElementById('inbox-settings-inline'); + if (!host) return; + try { + const r = await fetch('/api/inbox/settings'); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _inboxCache = Object.assign({}, _inboxCache, { settings: data }); + _inboxDraftRules = (data.rules || []).map(x => Object.assign({}, x)); + host.innerHTML = inboxSettingsFormHtml(data, true); + inboxRenderRules(host.querySelector('.inbox-settings-form')); + } catch (e) { + host.innerHTML = '<p class="kc-error muted-xs">Inbox settings unavailable: ' + escapeHtml(e.message) + '</p>'; + } + } + function inboxAddRule() { + _inboxDraftRules = inboxReadRules(); + _inboxDraftRules.push({ source: '', title_prefix: 'RFC:', tag: '', destination: 'spek_rfc', repo: '' }); + inboxRenderRules(); + } + function inboxRemoveRule(i) { + _inboxDraftRules = inboxReadRules(); + _inboxDraftRules.splice(Number(i), 1); + inboxRenderRules(); + } + async function inboxSaveSettings() { + const root = inboxSettingsRoot(); + if (!root) return; + const errEl = inboxSettingsField(root, 'error'); + const rules = inboxReadRules(root).map(r => { const o = {}; for (const k of Object.keys(r)) if (r[k]) o[k] = r[k]; return o; }); + const payload = { rules, spek_repo: (inboxSettingsField(root, 'spek-repo') || {}).value || '', dibs_url: (inboxSettingsField(root, 'dibs-url') || {}).value || '' }; + try { + const r = await fetch('/api/inbox/settings', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(payload) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + if (root.dataset.inboxForm !== 'inline') kbCloseModal(); + showToast('Inbox settings saved', 'success'); + await fetchInbox(); + } catch (e) { + if (errEl) errEl.textContent = e.message; else showToast('Save failed: ' + e.message, 'error'); + } + } + + // Hosted-hive Plaud sign-in completion: Plaud's OAuth client only redirects + // to localhost:8199, so the operator pastes either the redirect URL the + // browser landed on or the tokens-mcp.json contents; the hive finishes + // the login in-pod. + async function kbPlaudPasteComplete(name) { + const el = document.getElementById('kb-plaud-paste-' + name); + const raw = el ? el.value.trim() : ''; + if (!raw) { showToast('Paste the redirect URL or the tokens JSON first.', 'error'); return; } + const body = raw.startsWith('{') ? { tokens: raw } : { redirect_url: raw }; + try { + const r = await fetch('/api/knowledge/connectors/' + encodeURIComponent(name) + '/auth/complete', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + const data = await r.json().catch(() => ({})); + if (!r.ok) throw new Error(data.error || ('HTTP ' + r.status)); + _kbPlaudAuth[name] = data; + kbRenderIntegrationsModal(); + showToast(data.connected ? 'Plaud connected' + (data.account ? ' as ' + data.account : '') : 'Plaud sign-in submitted; click Check.', data.connected ? 'success' : 'info'); + fetchInbox(); + } catch (e) { + showToast('Plaud sign-in failed: ' + e.message, 'error'); + } + } + function kbPlaudPasteHtml(name) { + const n = escapeHtml(name); + const a = _kbPlaudAuth[name] || {}; + const openInbox = '<button type="button" class="hv-btn btn-secondary btn-sm" data-action="ocNavigate" data-arg0="inbox-section">📥 Open Inbox</button>'; + if (a.connected) return '<div class="kc-actions mt-2">' + openInbox + '<span class="muted-xs">Recordings are triaged in the Inbox, not here.</span></div>'; + return '<details class="kb-plaud-paste mt-2"><summary class="muted-xs">Hive runs on a server? Finish sign-in by pasting</summary>' + + '<p class="muted-xs">Plaud only redirects to <code>localhost:8199</code>. Open the authorization link, approve, then paste the <code>localhost:8199/auth/callback?…</code> URL your browser lands on (within 10 minutes). Alternatively paste the contents of <code>~/.plaud/tokens-mcp.json</code> from a laptop where <code>plaud-mcp</code> is signed in.</p>' + + '<textarea class="kb-search-box" id="kb-plaud-paste-' + n + '" rows="3" placeholder="http://localhost:8199/auth/callback?code=… or {"access_token": …}"></textarea>' + + '<div class="kc-actions mt-2"><button type="button" class="hv-btn btn-secondary btn-sm" data-action="kbPlaudPasteComplete" data-arg0="' + n + '">Submit</button>' + openInbox + '</div></details>'; + } // ── Vault Management ── function kbOpenVaults() { @@ -50826,6 +51326,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> ocUpdateSidebarAgents(); dashboardRenderSection('debug', ['debug-section'], () => renderDebugSection(), data); dashboardRenderSection('faq', ['faq-section'], () => renderFAQ(), data); + dashboardRenderSection('inbox', ['inbox-section'], () => renderInbox(), data); initializeDashboardSectionCards(); // Restore persisted collapsed state for all collapsible sections const COLLAPSIBLE_SECTION_IDS = [ @@ -50834,7 +51335,7 @@ <h3 class="modal-h">How Agents Use Knowledge (Priming)</h3> 'advisory-section', 'advisory-digest-section', 'hive-advice-section', 'acmm-reco-section', 'lifecycle-section', 'pr-throughput-section', 'overview-section', 'governor', 'token-panel', 'cost-panel', 'repos-section', 'acmm-eval-section', 'fleet-report-section', 'audit-section', 'review-queue-section', 'nous-section', 'inception-section', - 'knowledge-section', 'search-section', 'contributors-section', 'debug-section', 'logs-section', + 'knowledge-section', 'search-section', 'inbox-section', 'contributors-section', 'debug-section', 'logs-section', 'activity-monitor-section', 'agents-section', 'agent-activity-section', 'faq-section' ]; COLLAPSIBLE_SECTION_IDS.forEach(applySectionCollapse); @@ -57956,7 +58457,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> // which is read-only hub-synced user/role data. It sits next to Model // Gateways because both are backend configuration. const GOVERNOR_BOB_TAB = 'Bob'; - const GOVERNOR_CONFIG_TABS = ['General', 'Appearance', 'Dashboard', 'Agents', 'Thresholds', 'Labels', 'Repos', 'Workflows', 'Budget', 'Notifications', 'Channels', 'Health', 'Sensing', 'Logging', 'Features', 'Extensions', 'Knowledge', 'Project Observability', 'Advisory', 'Smart classifier', 'Work Source', 'Hub', 'Forge App', 'Model Gateways', GOVERNOR_BOB_TAB, 'Variables', 'Security', 'Compliance']; + const GOVERNOR_CONFIG_TABS = ['General', 'Appearance', 'Dashboard', 'Agents', 'Thresholds', 'Labels', 'Repos', 'Workflows', 'Budget', 'Notifications', 'Channels', 'Health', 'Sensing', 'Logging', 'Features', 'Extensions', 'Knowledge', 'Inbox', 'Project Observability', 'Advisory', 'Smart classifier', 'Work Source', 'Hub', 'Forge App', 'Model Gateways', GOVERNOR_BOB_TAB, 'Variables', 'Security', 'Compliance']; const PIPELINE_STAGES = ['resolve-beads', 'track-prs', 'stale-check', 'repo-scan', 'coverage-gate', 'prompt-compose', 'budget-check', 'api-collect', 'final-compose']; const PIPELINE_STAGE_TIPS = { 'resolve-beads': 'Fetch and inject knowledge beads (facts, patterns, gotchas) relevant to the agent\'s current task context.', @@ -58019,6 +58520,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> { tab: 'Labels', section: 'Labels', label: 'Adoption of a maintainer\'s PR', key: 'governor.labels.stage.adoption.maintainer_policy', text: 'adoption maintainer policy adopt ask never maintainer own pr clanker disconnected offline adopt action governor.labels.stage.adoption.maintainer_policy' }, { tab: 'Labels', section: 'Labels', label: 'Contributor nudge', key: 'governor.labels.stage.contributor_nudge.enabled', text: 'contributor nudge returned-to-contributor red ci fix-pr relay interrupt renudge presence window governor.labels.stage.contributor_nudge.enabled governor.labels.stage.contributor_nudge.renudge_interval auto_merge.nudge_repeat_after governor.labels.stage.contributor_nudge.ci_failure_cooldown governor.labels.stage.presence_window' }, { tab: 'Labels', section: 'Labels', label: 'Adopt red contributor PRs', key: 'governor.labels.stage.adoption.enabled', text: 'adoption adopted-by-hive stage 074 red contributor pr grace online offline supersede takeover in place after offline unregistered heartbeat governor.labels.stage.adoption.enabled governor.labels.stage.contributor_grace_online governor.labels.stage.contributor_grace_offline governor.labels.stage.adoption.after_offline governor.labels.stage.adoption.after_unregistered governor.labels.stage.adoption.in_place governor.labels.stage.adoption.strict_attribution strict attribution governor.labels.stage.adoption.auto_merge auto merge adopted head hive authority governor.labels.stage.adoption.supersede.close_first close first superseded reopen copy failed governor.labels.stage.adoption.close_superseded close superseded on adopt on merge clanker authored opt-in label governor.labels.stage.adoption.offline_grace offline grace rebase failed twice fixer resolving conflicts governor.labels.stage.adoption.rebase_attempt_timeout rebase attempt timeout' }, + { tab: 'Inbox', section: 'Inbox', label: 'Inbox routing', key: 'inbox.spek.repo', lazy: true, text: 'inbox voice recordings plaud wispr default rfc repository spek dibs url auto-routing rules inbox.spek.repo inbox.dibs.url inbox.rules' }, { tab: 'Hub', section: 'Contributor Relay', label: 'Fix budget', key: 'contribute.max_parallel_fixes', text: 'fix budget parallel fixes focus window red pr fix-pr preempt spawn contribute.max_parallel_fixes contribute.min_focus_window' }, { tab: 'Logging', section: 'Logging', label: 'Governor logging', key: 'governor.logging.max_size_mb', text: 'governor logging directory level compression max size age backups governor.logging.compress governor.logging.dir governor.logging.level governor.logging.max_age_days governor.logging.max_backups governor.logging.max_size_mb' }, { tab: 'Advisory', section: 'Advisory', label: 'Advisory digest', key: 'governor.advisory.max_findings', text: 'advisory digest max findings show all staleness pr autoclose target linear issue retry backoff update interval rewrite floor changed digest governor.advisory.update_interval_s governor.advisory.digest_min_interval' }, @@ -58063,6 +58565,8 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> { tab: 'Thresholds', section: 'Thresholds', label: 'Delivery SLA targets', key: 'sla.targets', text: 'delivery sla targets sla.targets.start_to_pr_h sla.targets.pr_to_merge_h sla.targets.open_to_merge_h hours sparkline overview stale in-progress' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 connector', key: 'knowledge.connectors', lazy: true, text: 'context7 connector api key env file reference library ids topic validate integrations knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Context7 auto-import', key: 'knowledge.context7.auto_import', lazy: true, text: 'context7 auto import startup libraries knowledge.context7.auto_import[].id knowledge.context7.auto_import[].query' }, + { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Plaud voice notes connector', key: 'knowledge.connectors', lazy: true, text: 'plaud voice notes recordings transcript summary mcp oauth connect npx @plaud-ai/mcp command since_days max_files include_audio tags knowledge.connectors' }, + { tab: 'Knowledge', section: 'Knowledge Integrations', label: 'Wispr Flow dictation connector', key: 'knowledge.connectors', lazy: true, text: 'wispr flow wisprflow dictation voice history flow.sqlite sqlite export path since_days max_items min_words group apps include_raw tags knowledge.connectors' }, { tab: 'Knowledge', section: 'Knowledge pipeline', label: 'Bead synthesizer, curator, code maps and primer', key: 'knowledge.bead_synthesizer.enabled', lazy: true, text: 'knowledge pipeline bead synthesizer wiki facts schedule hourly daily confidence target layer facts per cycle api calls vault retention archive beads high priority dependents curator auto promote threshold promotion from to code maps repository layer max bytes primer facts merge strategy engine public read-only mcp knowledge.engine knowledge.public.enabled knowledge.bead_synthesizer.enabled knowledge.bead_synthesizer.schedule knowledge.bead_synthesizer.min_confidence knowledge.bead_synthesizer.target_layer knowledge.bead_synthesizer.max_facts_per_cycle knowledge.bead_synthesizer.max_api_calls_per_cycle knowledge.bead_synthesizer.vault_path publish to repository knowledge.bead_synthesizer.publish.enabled knowledge.bead_synthesizer.publish.repo knowledge.bead_synthesizer.publish.path knowledge.bead_synthesizer.publish.base_branch knowledge.bead_synthesizer.publish.mode knowledge.bead_synthesizer.publish.cadence knowledge.bead_synthesizer.retention_policy.max_beads knowledge.bead_synthesizer.retention_policy.archive_after_synth_days knowledge.bead_synthesizer.retention_policy.high_priority_retain_days knowledge.bead_synthesizer.retention_policy.preserve_with_deps knowledge.curator.enabled knowledge.curator.schedule knowledge.curator.auto_promote_threshold knowledge.curator.promote_from knowledge.curator.promote_to knowledge.code_maps.enabled knowledge.code_maps.schedule knowledge.code_maps.layer knowledge.code_maps.max_bytes knowledge.code_maps.vault_path knowledge.primer.max_facts knowledge.primer.max_bytes knowledge.primer.merge_strategy' }, { tab: 'Forge App', section: 'Forge App', label: 'GitHub API quota, PR scans and authorship', key: 'github.api_reserve', lazy: true, text: 'github api quota reserve critical agent reserve floor pr detail cache ttl etag cache size mb entries conditional request 304 graphql batch page size app authored prs bot signed commits forge api url docs installation oauth client token github.api_reserve github.api_critical github.agent_reserve_floor github.etag_cache_max_mb github.etag_cache_max_entries github.pr_detail_ttl_s github.graphql_pr_batch github.graphql_pr_batch_page_size github.app_authored_prs github.app_signed_commits github.forge github.api_url github.docs_installation_id github.oauth_client_id github.token github.key_file' }, ]; @@ -59114,6 +59618,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> if (tabId === 'Features') loadLinearAgentStatus(); // The Features tab also hosts the advisor lane settings panel. if (tabId === 'Features') loadAdvisorSettings(); + if (tabId === 'Inbox') loadInboxSettingsInline(); // The same tab carries the upstream-watch editor and divergence view // (#9969, #11211); it loads from its own endpoint. if (tabId === 'Features') loadUpstreamWatch(); @@ -59330,6 +59835,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> case 'Features': return renderGovFeatures(d.features || {}, d.auto_merge || {}, d.review || {}); case 'Extensions': return renderGovExtensions(d.features || {}); case 'Knowledge': return renderGovKnowledgeSharing(); + case 'Inbox': return renderGovInbox(); case 'Project Observability': return renderGovProjectObservability(d.project_observability || {}); case 'Advisory': return renderGovAdvisory(d.advisory || {}); case 'Smart classifier': return renderGovSmartClassifier(d.classifier || {}); @@ -59355,7 +59861,7 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> } // ---- Settings → Knowledge → Connectors (#11075) ---- - const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'repo-wiki', 'sharepoint']; + const KB_CONNECTOR_TYPE_ORDER = ['confluence', 'context7', 'document', 'git', 'github-wiki', 'google-drive', 'notion', 'plaud', 'repo-wiki', 'sharepoint', 'wisprflow']; const KC_TYPE_INFO = { 'confluence': { label: 'Confluence', description: 'Sync Confluence Cloud or Data Center pages into knowledge facts.' }, 'context7': { label: 'Context7', description: 'Sync documentation for default Context7 libraries. The API key is a reference to an environment variable or secret file; the value is never shown.' }, @@ -59364,8 +59870,10 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> 'github-wiki': { label: 'GitHub Wiki', description: 'Clone and index one or more GitHub wiki repositories.' }, 'google-drive': { label: 'Google Drive', description: 'Sync Google Docs, Sheets, markdown, and text files from Drive folders or shared drives.' }, 'notion': { label: 'Notion', description: 'Sync Notion pages and database rows visible to an internal integration.' }, + 'plaud': { label: 'Plaud voice notes', description: 'Sync Plaud Note recordings (Plaud summary + transcript) through the free official Plaud MCP server. Sign-in is Plaud OAuth in your browser; hive never stores the token.' }, 'repo-wiki': { label: 'Repo wiki', description: 'Sync .hive/wiki (or another directory) from one or more GitHub repositories.' }, - 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' } + 'sharepoint': { label: 'SharePoint / OneDrive', description: 'Sync markdown, text, and HTML files from SharePoint document libraries or OneDrive drives.' }, + 'wisprflow': { label: 'Wispr Flow dictation', description: 'Sync your Wispr Flow dictation history (what you spoke, when, into which app) from the desktop app\'s local flow.sqlite or an export folder. Read-only, no sign-in, $0 beyond your Wispr Flow plan.' } }; const KC_TYPE_ICONS = { // Vendor marks: simple-icons (CC0 1.0) paths, 24x24 viewBox, rendered with currentColor. @@ -59379,6 +59887,8 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> // Generic glyphs (no vendor): document sheet and open book, drawn in-house. 'context7': 'M4 4h3v16H4zm5 0h3v16H9zm5.2.6 2.9-.8 3.8 15.5-2.9.8z', 'document': 'M14 2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12a2 2 0 0 0 2-2V8zm4 18H6V4h7v5h5zM8 12h8v1.6H8zm0 3.2h8v1.6H8zm0 3.2h5V20H8z', + 'plaud': 'M12 2a3.5 3.5 0 0 0-3.5 3.5v6a3.5 3.5 0 0 0 7 0v-6A3.5 3.5 0 0 0 12 2zm-6 9.5h1.8a4.2 4.2 0 0 0 8.4 0H18a6 6 0 0 1-5.1 5.93V20h3v1.8H8.1V20h3v-2.57A6 6 0 0 1 6 11.5z', + 'wisprflow': 'M3 10h2v4H3zm4-3h2v10H7zm4-5h2v20h-2zm4 5h2v10h-2zm4 3h2v4h-2z', 'repo-wiki': 'M4 3h7.2c1.1 0 2 .55 2.8 1.3.8-.75 1.7-1.3 2.8-1.3H20v16h-3.2c-1 0-1.95.4-2.8 1.1V22h-4v-1.9c-.85-.7-1.8-1.1-2.8-1.1H4zm2 2v12h1.2c1.3 0 2.5.4 3.6 1.1V6.2C10.4 5.55 9.6 5 8.8 5zm12 0h-1.2c-.8 0-1.6.55-2 1.2v12.1c1.1-.7 2.3-1.1 3.6-1.1H18z' }; function kbIntegrationIcon(type) { @@ -59394,8 +59904,10 @@ <h3 style="color:var(--amber);text-align:center">Configure LiteLLM</h3> 'github-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'master']], 'google-drive': [['folder_ids', 'Folder IDs', 'folder123, folder456'], ['shared_drive_ids', 'Shared drive IDs', 'drive123'], ['include_mime', 'Include MIME groups', 'docs,md,txt']], 'notion': [['root_page_ids', 'Root page IDs', '1a2b3c4d5e6f47a8b9c0d1e2f3a4b5c6'], ['database_ids', 'Database IDs', '0f1e2d3c-4b5a-4968-8776-655443322110'], ['include_archived', 'Include archived', 'false'], ['max_pages', 'Max pages', '2000'], ['full_sync_every', 'Full sync every', '10']], + 'plaud': [['command', 'MCP command', 'npx -y @plaud-ai/mcp@latest'], ['since_days', 'Days back on first sync', '30'], ['max_files', 'Max recordings per sync', '200'], ['include_audio', 'Include 24h audio link', 'false'], ['query', 'Name filter (optional)', 'standup'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['home', 'Token store HOME (optional)', '/data/knowledge/connector-state/plaud/home']], 'repo-wiki': [['repos', 'Repositories (comma-separated)', 'org/repo-a, org/repo-b'], ['branch', 'Branch', 'v5'], ['dir', 'Wiki directory', '.hive/wiki']], - 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']] + 'sharepoint': [['drive_ids', 'Drive IDs', 'b!drive-id'], ['site_ids', 'Site IDs', 'contoso.sharepoint.com,site-id,web-id'], ['folder_path', 'Folder path', '/Docs/Sub'], ['include_files', 'Include file extensions', '.md,.txt,.html']], + 'wisprflow': [['path', 'flow.sqlite or export folder (empty = app default)', '/Users/me/Library/Application Support/Wispr Flow/flow.sqlite'], ['since_days', 'Days back on first sync', '30'], ['max_items', 'Max dictations per sync', '500'], ['min_words', 'Skip dictations shorter than (words)', '3'], ['group', 'Fact per dictation or day', 'dictation'], ['apps', 'Only these apps (comma-separated, optional)', 'slack, obsidian, vscode'], ['include_raw', 'Include raw ASR text', 'false'], ['tags', 'Extra tags (comma-separated)', 'voice, ideas'], ['sqlite3', 'sqlite3 command (optional)', 'sqlite3']] }; let _kcState = { loaded: false, error: '', connectors: [], rows: [], types: [], layers: ['personal', 'project', 'org', 'community'], runtime: false, publish: null }; @@ -63143,6 +63655,9 @@ <h4 class="settings-subsection-title">Hub settings</h4> <div class="config-toggle-switch ${g.sandboxEnabled ? 'on' : ''}" id="cfg-agent-sandbox" data-action="toggleConfigSwitch" data-section="general" data-key="sandboxEnabled"></div> <span class="config-toggle-label">Run in sandbox ${settingHelpMark('sandbox-isolation', '', 'Run in sandbox', `Opt this agent into the credential-free sandbox runner. Requires the hive-wide sandbox gate in Governor → Security to be enabled.`)}${g.sandboxEffective ? ' ' + statusBadgeHTML('ready', 'sandboxed', '', 'done') : ''}</span> </div> + <details class="settings-subsection-details"><summary class="settings-subsection-summary">Substrate sandbox overrides</summary> + ${substrateSandboxFields('general', g.sandboxSubstrate)} + </details> <div class="config-toggle"> <div class="config-toggle-switch ${g.clearOnKick !== false ? 'on' : ''}" id="cfg-clear-on-kick" data-action="toggleConfigSwitch" data-section="general" data-key="clearOnKick"></div> <span class="config-toggle-label">Clear on Kick ${settingHelpMark('agent-configuration', 'anatomy-of-an-agent', 'Clear on Kick', `Send /clear before each governor kick to reset conversation context. Prevents stale context buildup across runs. On by default for all agents.`)}</span> @@ -70544,6 +71059,69 @@ <h4 style="margin:18px 0 8px">Escalation Breaker</h4> markDirty('security', key, n); } + const SUBSTRATE_SANDBOX_FIELDS = [ + { key: 'endpoint', label: 'Endpoint', kind: 'text', global: 'agent_sandbox.substrate.endpoint', agent: 'agents.<key>.sandbox.substrate.endpoint', help: `Substrate Control API address (host:port or an http(s):// URL). Empty uses api.ate-system.svc:443.` }, + { key: 'atespace', label: 'Atespace', kind: 'text', global: 'agent_sandbox.substrate.atespace', agent: 'agents.<key>.sandbox.substrate.atespace', help: `Substrate atespace (its tenant namespace, not a Kubernetes namespace) actors are created in. Empty uses hive.` }, + { key: 'token_env', label: 'Token env', kind: 'text', global: 'agent_sandbox.substrate.token_env', agent: 'agents.<key>.sandbox.substrate.token_env', help: `Environment variable holding a Control API bearer token. Empty reads the token file instead.` }, + { key: 'token_file', label: 'Token file', kind: 'text', global: 'agent_sandbox.substrate.token_file', agent: 'agents.<key>.sandbox.substrate.token_file', help: `Projected ServiceAccount token file hive authenticates with. Empty uses /var/run/secrets/ate/token.` }, + { key: 'ca_file', label: 'CA file', kind: 'text', global: 'agent_sandbox.substrate.ca_file', agent: 'agents.<key>.sandbox.substrate.ca_file', help: `PEM bundle the Control API serving certificate chains to. Empty uses the projected CA when present, else the system roots.` }, + { key: 'insecure_tls', label: 'Insecure TLS', kind: 'bool', global: 'agent_sandbox.substrate.insecure_tls', agent: 'agents.<key>.sandbox.substrate.insecure_tls', help: `Skip Control API and router certificate verification. Dev only.` }, + { key: 'timeout_s', label: 'Timeout (seconds)', kind: 'int', global: 'agent_sandbox.substrate.timeout_s', agent: 'agents.<key>.sandbox.substrate.timeout_s', help: `Bounds one actor from create to delete. 0 uses the sandbox timeout.` }, + { key: 'router_url', label: 'Router URL', kind: 'text', global: 'agent_sandbox.substrate.router_url', agent: 'agents.<key>.sandbox.substrate.router_url', help: `atenet-router CONNECT listener hive reaches the actor bridge through. Empty uses the in-cluster default.` }, + { key: 'router_client_bundle', label: 'Router client bundle', kind: 'text', global: 'agent_sandbox.substrate.router_client_bundle', agent: 'agents.<key>.sandbox.substrate.router_client_bundle', help: `podCertificate bundle the router authenticates hive with. Empty uses the default path.` }, + { key: 'router_ca_file', label: 'Router CA file', kind: 'text', global: 'agent_sandbox.substrate.router_ca_file', agent: 'agents.<key>.sandbox.substrate.router_ca_file', help: `CA the router serving certificate chains to. Empty uses the default path.` }, + { key: 'bridge_port', label: 'Bridge port', kind: 'int', global: 'agent_sandbox.substrate.bridge_port', agent: 'agents.<key>.sandbox.substrate.bridge_port', help: `Port the in-actor bridge listens on. 0 uses 8477.` }, + { key: 'image', label: 'Image', kind: 'text', global: 'agent_sandbox.substrate.image', agent: 'agents.<key>.sandbox.substrate.image', help: `Digest-pinned sandbox image (…@sha256:…) actors run. Substrate refuses tag-only references.` }, + { key: 'bridge_command', label: 'Bridge command', kind: 'lines', global: 'agent_sandbox.substrate.bridge_command[]', agent: 'agents.<key>.sandbox.substrate.bridge_command[]', help: `Actor entrypoint that serves the bridge. One argument per line; empty uses the crux substrate-bridge subcommand.` }, + { key: 'cpu_millicores', label: 'CPU (millicores)', kind: 'int', global: 'agent_sandbox.substrate.cpu_millicores', agent: 'agents.<key>.sandbox.substrate.cpu_millicores', help: `ActorTemplate CPU limit. 0 uses 2000.` }, + { key: 'memory_bytes', label: 'Memory (bytes)', kind: 'int', global: 'agent_sandbox.substrate.memory_bytes', agent: 'agents.<key>.sandbox.substrate.memory_bytes', help: `ActorTemplate memory limit. 0 uses 4 GiB.` }, + { key: 'worker_selector', label: 'Worker selector', kind: 'map', global: 'agent_sandbox.substrate.worker_selector.<key>', agent: 'agents.<key>.sandbox.substrate.worker_selector.<key>', help: `Pins actors to WorkerPools carrying these labels, one key=value per line. Empty means any pool.` }, + { key: 'snapshot_on_idle', label: 'Snapshot on idle', kind: 'bool', global: 'agent_sandbox.substrate.snapshot_on_idle', agent: 'agents.<key>.sandbox.substrate.snapshot_on_idle', help: `Suspend the actor with a snapshot after the kick instead of deleting it. Only the latest snapshot per agent is kept while hive runs (the previous one is deleted on the next suspend); each costs object storage.` }, + { key: 'allow_hosts', label: 'Allow hosts', kind: 'lines', global: 'agent_sandbox.substrate.allow_hosts[]', agent: 'agents.<key>.sandbox.substrate.allow_hosts[]', help: `Additional egress hosts (host or host:port, leading *. wildcard allowed), one per line, on top of the GitHub hosts.` }, + { key: 'policy_profile', label: 'Policy profile', kind: 'text', global: 'agent_sandbox.substrate.policy_profile', agent: 'agents.<key>.sandbox.substrate.policy_profile', help: `Operator-side policy profile recorded on the template.` }, + ]; + + /* substrateSandboxFields renders the Substrate runtime block for either the + global agent_sandbox.substrate (section 'security') or one agent's + sandbox.substrate override (section 'general'). */ + function substrateSandboxFields(section, sub) { + const dirtyKey = section === 'security' ? 'agentSandboxSubstrate' : 'sandboxSubstrate'; + sub = sub || {}; + return SUBSTRATE_SANDBOX_FIELDS.map(f => { + const yaml = section === 'security' ? f.global : f.agent; + const label = 'Substrate ' + f.label.toLowerCase(); + const head = '<div class="config-field"><label>' + esc(label) + ' ' + yamlHint(yaml) + ' ' + settingHelpMark('settings', 'security-and-access', label, esc(f.help)) + '</label>'; + const attrs = 'data-arg0="' + section + '" data-arg1="' + dirtyKey + '" data-arg2="' + f.key + '" data-arg3="' + f.kind + '"'; + const v = sub[f.key]; + if (f.kind === 'bool') { + return head + '<input type="checkbox"' + (v ? ' checked' : '') + ' data-change-action="markSubstrateSandbox" ' + attrs + ' data-arg-types="s,s,s,s,c"></div>'; + } + if (f.kind === 'int') { + return head + '<input type="number" min="0" value="' + Number(v || 0) + '" data-change-action="markSubstrateSandbox" ' + attrs + ' data-arg-types="s,s,s,s,v"></div>'; + } + if (f.kind === 'lines' || f.kind === 'map') { + const text = f.kind === 'map' + ? Object.keys(v || {}).sort().map(k => k + '=' + v[k]).join('\n') + : (v || []).join('\n'); + return head + '<textarea rows="3" class="mono" data-change-action="markSubstrateSandbox" ' + attrs + ' data-arg-types="s,s,s,s,v">' + esc(text) + '</textarea></div>'; + } + return head + '<input type="text" value="' + esc(v || '') + '" data-change-action="markSubstrateSandbox" ' + attrs + ' data-arg-types="s,s,s,s,v"></div>'; + }).join('\n'); + } + + function markSubstrateSandbox(section, dirtyKey, field, kind, value) { + const pending = _configState.dirty[section] && _configState.dirty[section][dirtyKey]; + const base = pending || ((_configState.data && _configState.data[section]) || {})[dirtyKey] || {}; + const next = Object.assign({}, base); + if (kind === 'bool') next[field] = !!value; + else if (kind === 'int') next[field] = Math.max(0, Math.floor(Number(value) || 0)); + else if (kind === 'lines') next[field] = String(value || '').split(/\r?\n/).map(s => s.trim()).filter(Boolean); + else if (kind === 'map') next[field] = parseSettingsMap(value); + else next[field] = String(value || '').trim(); + if (_configState.data && _configState.data[section]) _configState.data[section][dirtyKey] = next; + markDirty(section, dirtyKey, next); + } + function parseSecurityList(value) { return String(value || '').split(',').map(s => s.trim()).filter(Boolean); } @@ -70966,6 +71544,9 @@ <h4 style="margin:18px 0 8px">Escalation Breaker</h4> <div class="config-field"><label>Job TTL (seconds) ${yamlHint('agent_sandbox.job.ttl_seconds')} ${settingHelpMark('settings', 'security-and-access', 'Job TTL (seconds)', `Seconds a finished sandbox Job is kept. 0 uses the default.`)}</label><input type="number" min="0" value="${Number(sec.agentSandboxJobTtlSeconds || 0)}" data-change-action="markSecurityNumber" data-arg0="agentSandboxJobTtlSeconds" data-arg-types="s,v"></div> <div class="config-field"><label>Job workspace claim ${yamlHint('agent_sandbox.job.workspace_claim')} ${settingHelpMark('settings', 'security-and-access', 'Job workspace claim', `PVC holding the workspace for the job runtime. Required for runtime job.`)}</label><input type="text" value="${esc(sec.agentSandboxJobWorkspaceClaim || '')}" placeholder="PersistentVolumeClaim name" data-change-action="markDirty" data-arg0="security" data-arg1="agentSandboxJobWorkspaceClaim" data-arg-types="s,s,v"></div> <div class="config-field"><label>Job workspace claim mount ${yamlHint('agent_sandbox.job.workspace_claim_mount')} ${settingHelpMark('settings', 'security-and-access', 'Job workspace claim mount', `Where the workspace claim is mounted in the Job.`)}</label><input type="text" value="${esc(sec.agentSandboxJobWorkspaceClaimMount || '')}" placeholder="mount path" data-change-action="markDirty" data-arg0="security" data-arg1="agentSandboxJobWorkspaceClaimMount" data-arg-types="s,s,v"></div> + <details class="settings-subsection-details"><summary class="settings-subsection-summary">Substrate sandbox</summary> + ${substrateSandboxFields('security', sec.agentSandboxSubstrate)} + </details> </details>`; /* Backup encryption key: painted asynchronously from presence-only @@ -76316,7 +76897,7 @@ <h4>${model.editIndex === null ? 'Add Event trigger' : 'Edit Event trigger'}</h4 return; } - // Requirement #3: verify the App can access the repo's org. + // Verify the App installation covers this repository. setStatus('Checking GitHub App access…', 'var(--muted)'); try { const res = await fetch('/api/config/governor/repos/check-access', { @@ -76325,14 +76906,11 @@ <h4>${model.editIndex === null ? 'Add Event trigger' : 'Edit Event trigger'}</h4 }); const d = await res.json().catch(() => ({})); if (res.status === 409 && d && d.needsInstall) { - // App not installed on the new org: surface the correct per-forge - // install URL wired to the existing pending-install/recheck plumbing. - // Clicking Install flags the heartbeat (handleGitHubAppInstallClicked) - // exactly like the App-required banner; Re-check re-probes and, on - // success, completes the add. + // Guide the operator to install the App or update repository access, + // then re-check before adding the repo. const installBtn = (d.installUrl) - ? '<a href="' + esc(d.installUrl) + '" target="_blank" rel="noopener" data-action="markGitHubAppInstallClicked" style="color:var(--accent);font-weight:600">Install/authorize the App for ' + esc(d.org || '') + ' ↗</a>' - : '<span class="text-brand">This hive\'s GitHub Enterprise App slug is not configured, so an install link cannot be built — ask your operator to install the Hive App for ' + esc(d.org || '') + '.</span>'; + ? '<a href="' + esc(d.installUrl) + '" target="_blank" rel="noopener" data-action="markGitHubAppInstallClicked" style="color:var(--accent);font-weight:600">Manage App access for ' + esc(d.org || '') + ' ↗</a>' + : '<span class="text-brand">An installation link cannot be built — ask your operator to manage the Hive App access for ' + esc(d.org || '') + '.</span>'; setStatus('⚠ ' + esc(d.error || ('The App needs access to ' + val)) + '<br>' + installBtn + '  ·  <a href="#" data-action="recheckRepoAccessThenAdd" data-arg-types="e" style="color:var(--accent);font-weight:600">Re-check & add</a>', 'var(--amber)'); return; } @@ -76341,11 +76919,7 @@ <h4>${model.editIndex === null ? 'Add Event trigger' : 'Edit Event trigger'}</h4 return; } } catch (e) { - // Network error probing access is non-fatal: the server re-validates the - // single-host rule and the default-repo rule on save, so let the add - // proceed rather than hard-block on a transient dashboard hiccup. - _appendRepoAccepted(val); - setStatus('Added (App-access check could not be reached; the server will re-validate on save).', 'var(--muted)'); + setStatus('✕ Could not check GitHub App access. Please retry.', 'var(--red)'); return; } _appendRepoAccepted(val); diff --git a/src/pkg/dashboard/status_builder.go b/src/pkg/dashboard/status_builder.go index a8382ee7fa..daa1480968 100644 --- a/src/pkg/dashboard/status_builder.go +++ b/src/pkg/dashboard/status_builder.go @@ -1202,6 +1202,7 @@ func buildAgentsWithHidden(statuses map[string]*crux.Process, cfg *config.Config Sandboxed: agentCfg.SandboxEnabled(cfg.AgentSandbox), SandboxRun: sandboxRun, SandboxRuntime: effectiveSandboxRuntime(agentCfg, cfg), + SandboxWarnings: config.SandboxPreflightWarningsFor(effectiveSandboxRuntime(agentCfg, cfg)), Session: name, State: string(proc.State), Starting: proc.Starting && proc.State != crux.StateRunning, diff --git a/src/pkg/dashboard/status_stale_drop_test.go b/src/pkg/dashboard/status_stale_drop_test.go index 73c7abecb9..3ef712a878 100644 --- a/src/pkg/dashboard/status_stale_drop_test.go +++ b/src/pkg/dashboard/status_stale_drop_test.go @@ -97,21 +97,38 @@ func TestUpdateStatusIfFresh_StaleDropCounterAccumulates(t *testing.T) { } } -func TestUpdateStatusIfFresh_StarvationWarnsAfterThreshold(t *testing.T) { +func TestUpdateStatusIfFresh_StarvationPublishesAfterThreshold(t *testing.T) { var logBuf bytes.Buffer srv := NewServer(0, slog.New(slog.NewTextHandler(&logBuf, &slog.HandlerOptions{Level: slog.LevelInfo}))) for i := 0; i < staleDropStarvationThreshold-1; i++ { epoch := srv.BeginStatusSnapshot() srv.noteStatusMutation() - srv.UpdateStatusIfFresh(&StatusPayload{}, epoch) + if srv.UpdateStatusIfFresh(&StatusPayload{}, epoch) { + t.Fatalf("drop %d below the bound must not publish", i+1) + } } - if staleDropLoggedAt(logBuf.String(), "WARN") { - t.Fatalf("below the threshold must not WARN, got:\n%s", logBuf.String()) + if staleDropLoggedAt(logBuf.String(), "WARN") || strings.Contains(logBuf.String(), "level=WARN msg=\"publishing possibly-stale") { + t.Fatalf("below the bound must not WARN, got:\n%s", logBuf.String()) } + seq := srv.currentStatusSeq() epoch := srv.BeginStatusSnapshot() srv.noteStatusMutation() - srv.UpdateStatusIfFresh(&StatusPayload{}, epoch) - if !staleDropLoggedAt(logBuf.String(), "WARN") { - t.Fatalf("threshold-th consecutive drop must WARN, got:\n%s", logBuf.String()) + if !srv.UpdateStatusIfFresh(&StatusPayload{}, epoch) { + t.Fatalf("the bound-th stale build must publish, got:\n%s", logBuf.String()) + } + if got := srv.currentStatusSeq(); got != seq+1 { + t.Fatalf("statusSeq = %d, want %d", got, seq+1) + } + if n := strings.Count(logBuf.String(), "level=WARN msg=\"publishing possibly-stale"); n != 1 { + t.Fatalf("want exactly one bounded-staleness WARN, got %d:\n%s", n, logBuf.String()) + } + if staleDropLoggedAt(logBuf.String(), "WARN") { + t.Fatalf("per-drop line must never be WARN, got:\n%s", logBuf.String()) + } + srv.statusMu.RLock() + consecutive := srv.statusConsecutiveStaleDrops + srv.statusMu.RUnlock() + if consecutive != 0 { + t.Fatalf("counter must reset after the forced publish, got %d", consecutive) } } diff --git a/src/pkg/dashboard/webstatic/static_index_test.go b/src/pkg/dashboard/webstatic/static_index_test.go index 6cadc2590e..629a5f20df 100644 --- a/src/pkg/dashboard/webstatic/static_index_test.go +++ b/src/pkg/dashboard/webstatic/static_index_test.go @@ -420,7 +420,7 @@ func TestStaticDashboardDefaultLayoutOrder(t *testing.T) { } html := rec.Body.String() - defaultOrder := "'overview-section','governor','runs-section','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','search-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section'" + defaultOrder := "'overview-section','governor','runs-section','pr-throughput-section','repos-section','activity-monitor-section','knowledge-section','search-section','inbox-section','review-queue-section','contributors-section','advisory-section','token-panel','cost-panel','inception-section','acmm-eval-section','debug-section','audit-section','approvals-section','nous-section','logs-section','agents-section','agent-activity-section','faq-section'" if !strings.Contains(html, defaultOrder) { t.Fatalf("dashboard default layout/reset order no longer matches sidebar nav; missing %q", defaultOrder) } diff --git a/src/pkg/delegatedecision/issue_judgment.go b/src/pkg/delegatedecision/issue_judgment.go index e21278a332..16718210f7 100644 --- a/src/pkg/delegatedecision/issue_judgment.go +++ b/src/pkg/delegatedecision/issue_judgment.go @@ -94,20 +94,24 @@ func ParseSelfReport(title, body string) (SelfReport, bool) { // constants (pinned by TestSelfreportOpsClassesMatchFleetReport) without // importing that package's dependency tree into this leaf. var selfreportOpsRemedies = map[string]string{ - "agent_login_required": "re-authenticate the agent provider (run the provider's login on the spoke so the agent CLI has a current session)", - "inference_auth_failed": "renew the inference credentials or API key the spoke's agents use", - "data_disk_pressure": "free or grow the data disk on the spoke", - "github_app_token_mint_failed": "check the GitHub App key and installation the spoke is configured with", - "github_app_not_installed": "install the GitHub App on the organisation or repositories the spoke serves", - "github_app_repo_not_covered": "add the repository to the GitHub App installation the spoke uses", - "proxy_connect_dial_failed": "restore the spoke's outbound proxy or network path", - "proxy_tls_handshake_failed": "fix the spoke's proxy TLS trust (certificate or CA bundle)", - "crux_runtime_unavailable": "restart or re-deploy the crux runtime sidecar on the spoke", - "model_discovery_degraded": "check the model endpoint and credentials the spoke's provider is configured with", - "inference_upstream_error": "check the inference endpoint the spoke's provider points at", - "inference_caller_unidentified": "set the inference caller identity in the spoke's configuration", - "label_provisioning_failed": "grant the spoke's GitHub identity the label-admin permission it lacks", - "shared_ci_rerun_forbidden": "grant the spoke's GitHub identity the actions permission needed to re-run shared CI", + "agent_login_required": "re-authenticate the agent provider (run the provider's login on the spoke so the agent CLI has a current session)", + "inference_auth_failed": "renew the inference credentials or API key the spoke's agents use", + "data_disk_pressure": "free or grow the data disk on the spoke", + "github_app_token_mint_failed": "check the GitHub App key and installation the spoke is configured with", + "forge_unreachable": "the GitHub host did not answer from the pod — check cluster egress, DNS and any proxy or hostAliases in front of it, not the App", + "github_app_not_installed": "install the GitHub App on the organisation or repositories the spoke serves", + "github_app_repo_not_covered": "add the repository to the GitHub App installation the spoke uses", + "github_app_alerts_permission_missing": "grant Dependabot alerts read-only in the GitHub App settings and re-accept the permission on its installation", + "github_app_statuses_permission_missing": "grant Commit statuses read & write in the GitHub App settings and re-accept the permission on its installation", + "proxy_connect_dial_failed": "restore the spoke's outbound proxy or network path", + "proxy_plain_forward_failed": "check the requested HTTP host, DNS and outbound network path on the spoke", + "proxy_tls_handshake_failed": "fix the spoke's proxy TLS trust (certificate or CA bundle)", + "crux_runtime_unavailable": "restart or re-deploy the crux runtime sidecar on the spoke", + "model_discovery_degraded": "check the model endpoint and credentials the spoke's provider is configured with", + "inference_upstream_error": "check the inference endpoint the spoke's provider points at", + "inference_caller_unidentified": "set the inference caller identity in the spoke's configuration", + "label_provisioning_failed": "grant the spoke's GitHub identity the label-admin permission it lacks", + "shared_ci_rerun_forbidden": "grant the spoke's GitHub identity the actions permission needed to re-run shared CI", } // selfreportOpsClass reports whether a self-report class is an operator's to diff --git a/src/pkg/delegatedecision/issue_judgment_test.go b/src/pkg/delegatedecision/issue_judgment_test.go index 4c20d4367e..603bafd44f 100644 --- a/src/pkg/delegatedecision/issue_judgment_test.go +++ b/src/pkg/delegatedecision/issue_judgment_test.go @@ -27,6 +27,14 @@ func TestDecideIssueLikeTheOwner(t *testing.T) { text: []string{"@clubanderson", "re-authenticate the agent provider", "hosted-ibm-alchemy-logg-ptoo"}, actions: []ActionKind{ActionLabelsAdd, ActionRecommendOnly}, }, + { + name: "missing Dependabot alerts permission is operator action", + in: Input{Title: "[selfreport] github_app_alerts_permission_missing: scoped token mint", + Body: "<!-- hive:selfreport fp=x hive=spoke -->\nclass `github_app_alerts_permission_missing`\n| owner(s) | @clubanderson |\n| spoke | spoke |"}, + rule: "selfreport-operator-action", verdict: VerdictOperatorAction, + text: []string{"@clubanderson", "Dependabot alerts read-only", "re-accept", "spoke"}, + actions: []ActionKind{ActionLabelsAdd, ActionRecommendOnly}, + }, { name: "hive#15329 RFC on hive-filed proposals (human author) → recommend only, with a real recommendation", in: fixtureHive15329, rule: "hive-policy", verdict: VerdictRecommendOnly, @@ -160,7 +168,8 @@ func TestSelfreportOpsClassesMatchFleetReport(t *testing.T) { for _, c := range []string{ fleetreport.ClassAgentLoginRequired, fleetreport.ClassInferenceAuthFailed, fleetreport.ClassDataDiskPressure, fleetreport.ClassGitHubAppTokenMintFailed, fleetreport.ClassGitHubAppNotInstalled, fleetreport.ClassGitHubAppRepoNotCovered, - fleetreport.ClassProxyConnectDialFailed, fleetreport.ClassProxyTLSHandshakeFailed, fleetreport.ClassCruxRuntimeUnavailable, + fleetreport.ClassGitHubAppAlertsPermission, fleetreport.ClassGitHubAppStatusesPermission, fleetreport.ClassForgeUnreachable, + fleetreport.ClassProxyConnectDialFailed, fleetreport.ClassProxyPlainForwardFailed, fleetreport.ClassProxyTLSHandshakeFailed, fleetreport.ClassCruxRuntimeUnavailable, fleetreport.ClassModelDiscoveryDegraded, fleetreport.ClassInferenceUpstreamError, fleetreport.ClassInferenceCallerUnknown, fleetreport.ClassLabelProvisioningFailed, fleetreport.ClassSharedCIRerunForbidden, } { diff --git a/src/pkg/effects/effects.go b/src/pkg/effects/effects.go index a7066d3756..f57ef11014 100644 --- a/src/pkg/effects/effects.go +++ b/src/pkg/effects/effects.go @@ -28,6 +28,7 @@ var ( ErrDenied = cruxeffects.ErrDenied ErrNotApplied = cruxeffects.ErrNotApplied ErrNeedsReconciliation = cruxeffects.ErrNeedsReconciliation + ErrUnresolvedExpired = cruxeffects.ErrUnresolvedExpired ) type ( diff --git a/src/pkg/evidence/benchmarks/convert.go b/src/pkg/evidence/benchmarks/convert.go index 0f9f6c5966..6f52f14ab5 100644 --- a/src/pkg/evidence/benchmarks/convert.go +++ b/src/pkg/evidence/benchmarks/convert.go @@ -27,6 +27,9 @@ type Options struct { // SandboxRuntime resolves the runtime an agent's kicks ran under // ("none" when sandboxing is off). nil means "none" for every agent. SandboxRuntime func(agent string) string + // CostLookup attributes kick spend to a PR (author: the PR and the issues + // it closed; reviewer: the PR). nil leaves cost.basis "unknown". + CostLookup CostLookup } // Roles. @@ -133,6 +136,7 @@ func Convert(bundles []*evidence.Bundle, opts Options) []Record { rec.Model = Model{Provider: providerOf(a.Model), Name: strings.TrimSpace(a.Model)} rec.SandboxRuntime = runtimeOf(opts, a, name) rec.Task.Kind = kindOf(a.TaskKind, name) + rec.Cost = costOf(opts.CostLookup, authorCostRefs(latest)) out = append(out, rec) } if backend, model, ok := dominantReviewer(latest.Verdicts); ok { @@ -142,6 +146,7 @@ func Convert(bundles []*evidence.Bundle, opts Options) []Record { rec.Model = Model{Provider: providerOf(model), Name: model} rec.SandboxRuntime = SandboxRuntimeNone rec.Task.Kind = kindPRReview + rec.Cost = costOf(opts.CostLookup, prCostRefs(latest.Repo, latest.Number, costKindPR)) out = append(out, rec) } } @@ -149,6 +154,49 @@ func Convert(bundles []*evidence.Bundle, opts Options) []Record { return out } +const ( + costKindIssue = "issue" + costKindPR = "pr" +) + +// prCostRefs spells one ref both ways the kick path records it. +func prCostRefs(repo string, number int, kind string) []CostRef { + full := repo + "#" + strconv.Itoa(number) + refs := []CostRef{{Ref: full, Kind: kind}} + if _, short, ok := strings.Cut(repo, "/"); ok && short != "" { + refs = append(refs, CostRef{Ref: short + "#" + strconv.Itoa(number), Kind: kind}) + } + return refs +} + +// authorCostRefs lists the refs an authoring kick may have been launched +// against: the PR itself and every same-repo issue it closed. +func authorCostRefs(b *evidence.Bundle) []CostRef { + refs := prCostRefs(b.Repo, b.Number, costKindIssue) + for _, c := range b.IssueClosures { + repo := c.Repo + if repo == "" { + repo = b.Repo + } + if c.Issue > 0 { + refs = append(refs, prCostRefs(repo, c.Issue, costKindIssue)...) + } + } + return refs +} + +func costOf(lookup CostLookup, refs []CostRef) Cost { + if lookup != nil { + if c, ok := lookup(refs); ok && c.USDEst != nil && *c.USDEst > 0 { + if c.Basis == "" { + c.Basis = CostBasisTokensXPricing + } + return c + } + } + return Cost{Basis: CostBasisUnknown} +} + // maxACMM is the highest ACMM level the schema accepts. const maxACMM = 6 diff --git a/src/pkg/evidence/benchmarks/convert_test.go b/src/pkg/evidence/benchmarks/convert_test.go index b70a87d042..6bef3b93b8 100644 --- a/src/pkg/evidence/benchmarks/convert_test.go +++ b/src/pkg/evidence/benchmarks/convert_test.go @@ -28,9 +28,10 @@ func fixtureBundles() []*evidence.Bundle { {Perspective: "security", Model: "claude-fable-5.1", Backend: "copilot", Verdict: "approve", Confidence: 0.9}, {Perspective: "style", Model: "claude-fable-5.1", Backend: "copilot", Verdict: "comment", Confidence: 0.7}, }, - HumanActions: []evidence.Action{{Actor: "andy", Kind: "label", At: t0(2 * time.Hour)}}, - Merge: &evidence.MergeEvent{Actor: "hive", Method: "squash", At: t0(3 * time.Hour), OpenedAt: t0(-time.Hour)}, - CreatedAt: t0(2 * time.Hour), UpdatedAt: t0(3 * time.Hour), + HumanActions: []evidence.Action{{Actor: "andy", Kind: "label", At: t0(2 * time.Hour)}}, + IssueClosures: []evidence.IssueClosure{{Repo: "hivecommons/hive", Issue: 40, At: t0(3 * time.Hour)}}, + Merge: &evidence.MergeEvent{Actor: "hive", Method: "squash", At: t0(3 * time.Hour), OpenedAt: t0(-time.Hour)}, + CreatedAt: t0(2 * time.Hour), UpdatedAt: t0(3 * time.Hour), } private := &evidence.Bundle{ SchemaVersion: evidence.SchemaVersion, ID: "c", Repo: "acme/secret", Number: 7, @@ -53,6 +54,66 @@ func testOptions() Options { RepoConsent: true, RepoPublic: func(repo string) bool { return strings.HasPrefix(repo, "hivecommons/") }, SandboxRuntime: func(agent string) string { return "none" }, + CostLookup: fixtureCostLookup, + } +} + +// fixtureCostLookup mimics the launch-cost store: the governor kicked short +// "hive#40" (issue the PR closed), the swarm reviewed full "hivecommons/hive#42". +func fixtureCostLookup(refs []CostRef) (Cost, bool) { + spend := map[CostRef]float64{ + {Ref: "hive#40", Kind: "issue"}: 1.25, + {Ref: "hivecommons/hive#42", Kind: "pr"}: 0.40, + {Ref: "acme/secret#7", Kind: "issue"}: 0, // kick landed no usage + } + var usd float64 + n := 0 + for _, r := range refs { + if v, ok := spend[r]; ok { + usd += v + n++ + } + } + if n == 0 { + return Cost{Basis: CostBasisUnknown}, false + } + return Cost{Basis: CostBasisTokensXPricing, USDEst: &usd, Launches: n, PriceTableDate: "2026-11-09"}, true +} + +func TestConvertAttributesCost(t *testing.T) { + recs := Convert(fixtureBundles(), testOptions()) + author := byRole(recs, "hivecommons/hive", RoleAuthor) + if author == nil || author.Cost.Basis != CostBasisTokensXPricing || author.Cost.USDEst == nil || *author.Cost.USDEst != 1.25 { + t.Fatalf("author cost via closed issue short ref: %+v", author) + } + reviewer := byRole(recs, "hivecommons/hive", RoleReviewer) + if reviewer == nil || reviewer.Cost.USDEst == nil || *reviewer.Cost.USDEst != 0.40 || reviewer.Cost.Launches != 1 { + t.Fatalf("reviewer cost via full pr ref: %+v", reviewer) + } + private := byRole(recs, "", RoleAuthor) + if private == nil || private.Cost.Basis != CostBasisUnknown || private.Cost.USDEst != nil { + t.Fatalf("zero-usage kick must be unknown: %+v", private) + } + noLookup := testOptions() + noLookup.CostLookup = nil + for _, r := range Convert(fixtureBundles(), noLookup) { + if r.Cost.Basis != CostBasisUnknown { + t.Fatalf("nil lookup must yield unknown, got %+v", r.Cost) + } + } +} + +func TestCostRefsSpellBothForms(t *testing.T) { + b := &evidence.Bundle{Repo: "o/r", Number: 5, IssueClosures: []evidence.IssueClosure{{Issue: 3}}} + got := authorCostRefs(b) + want := []CostRef{{"o/r#5", "issue"}, {"r#5", "issue"}, {"o/r#3", "issue"}, {"r#3", "issue"}} + if len(got) != len(want) { + t.Fatalf("refs %v", got) + } + for i := range want { + if got[i] != want[i] { + t.Fatalf("ref %d = %v want %v", i, got[i], want[i]) + } } } diff --git a/src/pkg/evidence/benchmarks/record.go b/src/pkg/evidence/benchmarks/record.go index 50b2aaf0f9..875297febd 100644 --- a/src/pkg/evidence/benchmarks/record.go +++ b/src/pkg/evidence/benchmarks/record.go @@ -49,6 +49,7 @@ type Record struct { Task Task `json:"task"` Outcome Outcome `json:"outcome"` Quality *Quality `json:"quality,omitempty"` + Cost Cost `json:"cost"` } // Source says which hive emitted the record. @@ -109,6 +110,40 @@ type Quality struct { SentinelFindings *int `json:"sentinel_findings,omitempty"` } +// Cost basis values (schema cost.basis). A spoke never emits +// fleet_model_median — the index builder imputes that from published records. +const ( + CostBasisReported = "reported" + CostBasisTokensXPricing = "tokens_x_pricing" + CostBasisUnknown = "unknown" +) + +// Cost is the record's estimated spend. USDEst is hive's launch attribution +// priced with its public list-price table; it is an estimate, never a bill. +type Cost struct { + Basis string `json:"basis"` + USDEst *float64 `json:"usd_est,omitempty"` + InputTokens *int64 `json:"input_tokens,omitempty"` + OutputTokens *int64 `json:"output_tokens,omitempty"` + CacheReadTokens *int64 `json:"cache_read_tokens,omitempty"` + CacheWriteTokens *int64 `json:"cache_write_tokens,omitempty"` + // Launches counts the kicks whose spend was attributed to the record. + Launches int `json:"launches,omitempty"` + PriceTableDate string `json:"price_table_date,omitempty"` +} + +// CostRef is one work reference a kick may have named. Ref carries the +// spelling the kick path uses: issue refs are "repo#N" (governor), PR refs +// are "owner/repo#N" (review swarm); Convert asks for both. +type CostRef struct { + Ref string + Kind string // "issue" or "pr" +} + +// CostLookup sums the attributed spend of every ref and reports whether any +// usage landed. Spend of zero counts as no attribution. +type CostLookup func(refs []CostRef) (Cost, bool) + // recordID derives the stable id of one (hive, PR, role) triple so re-exports // overwrite rather than duplicate. func recordID(hiveHash, repo string, number int, role string) string { diff --git a/src/pkg/evidence/benchmarks/testdata/result.schema.json b/src/pkg/evidence/benchmarks/testdata/result.schema.json index c4ecd39ec6..87f9edd6bb 100644 --- a/src/pkg/evidence/benchmarks/testdata/result.schema.json +++ b/src/pkg/evidence/benchmarks/testdata/result.schema.json @@ -357,7 +357,28 @@ "cost": { "type": "object", "additionalProperties": false, + "description": "Estimated spend for this record. usd_est is ALWAYS an estimate (list prices, apportioned kicks); basis says how it was derived. Records with basis=unknown carry no usd_est and are excluded from cost aggregates.", "properties": { + "usd_est": { + "type": "number", + "minimum": 0, + "description": "Estimated USD for this PR/review." + }, + "basis": { + "type": "string", + "enum": [ + "reported", + "tokens_x_pricing", + "fleet_model_median", + "unknown" + ], + "description": "reported: the harness/gateway returned a billed figure. tokens_x_pricing: hive's per-kick token counts \u00d7 schema/pricing.json list prices, apportioned to this PR via launch attribution (kick refs \u2192 issue_closures/PR). fleet_model_median: no attribution for this PR; median tokens_x_pricing cost for the same (model, role) across the fleet. unknown: nothing available." + }, + "usd": { + "type": "number", + "minimum": 0, + "description": "Billed USD when basis=reported." + }, "input_tokens": { "type": "integer", "minimum": 0 @@ -374,25 +395,20 @@ "type": "integer", "minimum": 0 }, - "usd": { - "type": "number", - "minimum": 0 - }, - "basis": { - "type": "string", - "description": "How usd was computed (GET /api/cost: estimated from public price table, or apportioned from per-agent totals).", - "enum": [ - "estimated", - "apportioned", - "billed", - "unpriced" - ] + "launches": { + "type": "integer", + "minimum": 0, + "description": "Agent kicks attributed to this PR (tokens_x_pricing only)." }, "price_table_date": { "type": "string", - "format": "date" + "format": "date", + "description": "as_of of the price table used." } - } + }, + "required": [ + "basis" + ] }, "quality": { "type": "object", diff --git a/src/pkg/fleetreport/runtime.go b/src/pkg/fleetreport/runtime.go index 48d0baf0af..fb64d5f3c6 100644 --- a/src/pkg/fleetreport/runtime.go +++ b/src/pkg/fleetreport/runtime.go @@ -39,6 +39,7 @@ const ( ClassDataDiskPressure = crux.ClassDataDiskPressure ClassGitHubAppTokenMintFailed = crux.ClassGitHubAppTokenMintFailed ClassGitHubAppNotInstalled = crux.ClassGitHubAppNotInstalled + ClassForgeUnreachable = crux.ClassForgeUnreachable ClassUnexpectedHub5xx = "unexpected_http_5xx_from_hub" ClassUnclassifiedRepeated = "unclassified_error_repeated" // ClassAgentStall, ClassAgentCrashLoop and ClassCIPollNudge replace the @@ -68,34 +69,39 @@ const ( // Log-fed classes: faults that have no dedicated hook site and are recognised // from their WARN+ log line by the slog feed (see LogFedClass). const ( - ClassTmuxSessionCheckFailed = "tmux_session_check_failed" - ClassAgentPaneTimeout = "agent_pane_timeout" - ClassProxyTLSHandshakeFailed = "proxy_tls_handshake_failed" - ClassAgentDirSetupFailed = "agent_dir_setup_failed" - ClassReviewEvidenceWriteFail = "review_evidence_write_failed" - ClassFeedbackIssueRefreshFailed = "feedback_issue_refresh_failed" - ClassBeadFactIngestFailed = "bead_fact_ingest_failed" - ClassSelfReportClearedFailed = "selfreport_cleared_comment_failed" - ClassGitHubAppRepoNotCovered = "github_app_repo_not_covered" - ClassProxyConnectDialFailed = "proxy_connect_dial_failed" - ClassProxyPlainForwardRefused = "proxy_plain_forward_refused" - ClassInferenceUpstreamError = "inference_upstream_error" - ClassInferenceCallerUnknown = "inference_caller_unidentified" - ClassInferenceNoRoute = "inference_no_route" - ClassSHAPollCommitMsgUnavail = "sha_poll_commit_message_unavailable" - ClassKickFreshnessWithheld = "kick_freshness_withheld" - ClassModelsEndpointEmpty = "models_endpoint_empty" - ClassAgentLoginRequired = "agent_login_required" - ClassPRRequestHeadUnpushed = "pr_request_head_unpushed" - ClassIOScanClassifierUnavail = "ioscan_classifier_unavailable" - ClassGitHubFetchTransient = "github_fetch_transient" - ClassLabelProvisioningFailed = "label_provisioning_failed" - ClassAgentCLIRestartLoop = "agent_cli_restart_loop" - ClassAgentGitHubCapReached = "agent_github_api_cap_reached" - ClassCopilotCLIHung = "copilot_cli_hung" - ClassTranscriptScanSlow = "transcript_scan_slow" - ClassSharedCIRerunForbidden = "shared_ci_rerun_forbidden" - ClassIssueClaimEffectFailed = "issue_claim_effect_failed" + ClassTmuxSessionCheckFailed = "tmux_session_check_failed" + ClassAgentPaneTimeout = "agent_pane_timeout" + ClassProxyTLSHandshakeFailed = "proxy_tls_handshake_failed" + ClassAgentDirSetupFailed = "agent_dir_setup_failed" + ClassReviewEvidenceWriteFail = "review_evidence_write_failed" + ClassFeedbackIssueRefreshFailed = "feedback_issue_refresh_failed" + ClassBeadFactIngestFailed = "bead_fact_ingest_failed" + ClassSelfReportClearedFailed = "selfreport_cleared_comment_failed" + ClassGitHubAppRepoNotCovered = "github_app_repo_not_covered" + ClassProxyConnectDialFailed = "proxy_connect_dial_failed" + ClassProxyPlainForwardRefused = "proxy_plain_forward_refused" + ClassInferenceUpstreamError = "inference_upstream_error" + ClassInferenceCallerUnknown = "inference_caller_unidentified" + ClassInferenceNoRoute = "inference_no_route" + ClassSHAPollCommitMsgUnavail = "sha_poll_commit_message_unavailable" + ClassKickFreshnessWithheld = "kick_freshness_withheld" + ClassModelsEndpointEmpty = "models_endpoint_empty" + ClassAgentLoginRequired = "agent_login_required" + ClassPRRequestHeadUnpushed = "pr_request_head_unpushed" + ClassIOScanClassifierUnavail = "ioscan_classifier_unavailable" + ClassGitHubFetchTransient = "github_fetch_transient" + ClassLabelProvisioningFailed = "label_provisioning_failed" + ClassAgentCLIRestartLoop = "agent_cli_restart_loop" + ClassAgentGitHubCapReached = "agent_github_api_cap_reached" + ClassCopilotCLIHung = "copilot_cli_hung" + ClassTranscriptScanSlow = "transcript_scan_slow" + ClassSharedCIRerunForbidden = "shared_ci_rerun_forbidden" + ClassIssueClaimEffectFailed = "issue_claim_effect_failed" + ClassGitHubAppAlertsPermission = "github_app_alerts_permission_missing" + ClassGitHubAppStatusesPermission = "github_app_statuses_permission_missing" + ClassHubHeartbeatUnprovisioned = "hub_heartbeat_rejected_unprovisioned" + ClassProxyPlainForwardFailed = "proxy_plain_forward_failed" + ClassAgentLifecycleRepeated = "agent_lifecycle_repeated" // A restart or normal stop can invalidate a pending bootstrap without // indicating a restart storm; only repeated interruptions are reportable. ClassStartupKickInterrupted = "startup_kick_interrupted" @@ -170,53 +176,59 @@ type ClassSpec struct { // Classes is the runtime error allow-list. var Classes = map[string]ClassSpec{ - ClassRestartStorm: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"restart storm", "restart-storm"}}, - ClassKickFailedRepeatedly: {Component: "agent-runtime", Severity: "medium", MinCount: kickFailureStreak, Keywords: []string{"kick failed", "undeliverable kick", "kicks undeliverable", "re-kick backed off"}}, - ClassPanicRecovered: {Component: "dashboard", Severity: "high", MinCount: 1, Keywords: []string{"panic recovered", "recovered panic"}}, - ClassModelDiscoveryDegraded: {Component: "inference-gateway", Severity: "medium", MinCount: 1, MinDuration: degradedMinDuration, Keywords: []string{"model discovery degraded", "model discovery failed"}}, - ClassInferenceAuthFailed: {Component: "backend-auth", Severity: "high", MinCount: 1, Keywords: []string{"litellm 401", "inference 401", "inference 403", "authentication failure"}}, - ClassAdvisoryPostFailed: {Component: "github-client", Severity: "medium", MinCount: 1, Keywords: []string{"advisory post failed", "advisory digest"}}, - ClassUpgradeStalled: {Component: "agent-lifecycle", Severity: "high", MinCount: 1, Keywords: []string{"upgrade stalled", "upgrade stall"}}, - ClassDataDiskPressure: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"disk full", "disk pressure", "no space left"}}, - ClassGitHubAppTokenMintFailed: {Component: "github-client", Severity: "high", MinCount: 1, Keywords: []string{"token mint failed", "installation token"}}, - ClassGitHubAppNotInstalled: {Component: "github-client", Severity: "medium", MinCount: 1, Keywords: []string{"github app installation id is unset", "github app has no installation"}}, - ClassUnexpectedHub5xx: {Component: "dashboard", Severity: "medium", MinCount: hub5xxStreak, Keywords: []string{"hub 5", "hub returned 5"}}, - ClassUnclassifiedRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear}, - ClassAgentStall: {Component: "agent-runtime", Severity: "medium", MinCount: AgentStallNudgeStreak, Keywords: []string{"agent stall", "stalled after kick", "continue nudge"}}, - ClassAgentCrashLoop: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"crash loop", "crash-loop", "crash looping"}}, - ClassCIPollNudge: {Component: "agent-runtime", Severity: "low", MinCount: 1, Window: ciPollNudgeWindow, Keywords: []string{"ci-poll nudge", "polling ci", "stop-polling nudge"}}, - ClassIssueRequestStuck: {Component: "github-relay", Severity: "medium", MinCount: 1, Keywords: []string{"issue-request stuck", "issue request stuck", "issue-request watcher", "relay request stuck"}}, - ClassCruxRuntimeUnavailable: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"crux runtime unavailable", "crux server unreachable"}}, - ClassEffectLedgerOpStuck: {Component: "mutation-ledger", Severity: "medium", MinCount: 1, Keywords: []string{"effect ledger stuck", "ledger op stuck", "needs reconciliation", "mutation journal unknown"}}, - ClassTmuxSessionCheckFailed: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"tmux has-session failed"}}, - ClassStartupKickInterrupted: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"startup kick dropped: agent restarted or stopped while waiting"}}, - ClassAgentPaneTimeout: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"prompt timeout — dumping pane", "pane content at timeout", "startup kick dropped: cli never became ready", "startup kick dropped: cli never reached input prompt", "kick undeliverable"}}, - ClassProxyTLSHandshakeFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy client tls handshake failed", "transparent proxy tls handshake failed", "copilot sniff: client tls handshake failed"}}, - ClassAgentDirSetupFailed: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to create per-agent xdg dir", "failed to restore per-agent dir mode", "failed to restore interactive home mode", "failed to tighten per-agent dir mode", "failed to tighten interactive home mode", "failed to restore inference home mode", "failed to tighten inference home mode"}}, - ClassReviewEvidenceWriteFail: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"review evidence: could not write bundle"}}, - ClassFeedbackIssueRefreshFailed: {Component: "dashboard", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"feedback: issue refresh failed"}}, - ClassBeadFactIngestFailed: {Component: "knowledge", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to ingest bead fact"}}, - ClassSelfReportClearedFailed: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"fleet report: cleared comment failed"}}, - ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}}, - ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed", "copilot sniff: upstream dial failed"}}, - ClassProxyPlainForwardRefused: {Component: "github-proxy", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward refused"}}, - ClassInferenceCallerUnknown: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference request from unidentified caller refused"}}, - ClassInferenceNoRoute: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference no route"}}, - ClassInferenceUpstreamError: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference upstream error", "inference upstream failed", "inference translate request failed"}}, - ClassSHAPollCommitMsgUnavail: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"sha poll: commit message fetch non-200"}}, - ClassKickFreshnessWithheld: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"kick withheld: every ref failed the freshness check"}}, - ClassModelsEndpointEmpty: {Component: "dashboard", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"no models found from any endpoint"}}, - ClassAgentLoginRequired: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"login required detected", "watchdog: agent credential failure"}}, - ClassPRRequestHeadUnpushed: {Component: "github-relay", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"head branch is not on the remote"}}, - ClassIOScanClassifierUnavail: {Component: "inference-gateway", Severity: "medium", MinCount: 1, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"ioscan classifier failed closed", "ioscan classifier failed open", "ioscan fail-closed blocked kick", "ioscan fail-closed blocked held-pr coordination kick"}}, - ClassAgentGitHubCapReached: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"agent github api hourly cap reached"}}, - ClassGitHubFetchTransient: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"channel commit date: fetch failed"}}, - ClassLabelProvisioningFailed: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue stage labels: provisioning failed"}}, - ClassAgentCLIRestartLoop: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"agent cli crashed or stuck on consent screen"}}, - ClassCopilotCLIHung: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"copilot hung with no cli prompt", "diagnostic: timed out waiting for copilot error output"}}, - ClassSharedCIRerunForbidden: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"shared-ci rerun forbidden"}}, - ClassIssueClaimEffectFailed: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue-claims: github side effect failed"}}, - ClassTranscriptScanSlow: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"transcript activity scan outlived its interval"}}, + ClassRestartStorm: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"restart storm", "restart-storm"}}, + ClassKickFailedRepeatedly: {Component: "agent-runtime", Severity: "medium", MinCount: kickFailureStreak, Keywords: []string{"kick failed", "undeliverable kick", "kicks undeliverable", "re-kick backed off"}}, + ClassPanicRecovered: {Component: "dashboard", Severity: "high", MinCount: 1, Keywords: []string{"panic recovered", "recovered panic"}}, + ClassModelDiscoveryDegraded: {Component: "inference-gateway", Severity: "medium", MinCount: 1, MinDuration: degradedMinDuration, Keywords: []string{"model discovery degraded", "model discovery failed"}}, + ClassInferenceAuthFailed: {Component: "backend-auth", Severity: "high", MinCount: 1, Keywords: []string{"litellm 401", "inference 401", "inference 403", "authentication failure"}}, + ClassAdvisoryPostFailed: {Component: "github-client", Severity: "medium", MinCount: 1, Keywords: []string{"advisory post failed", "advisory digest"}}, + ClassUpgradeStalled: {Component: "agent-lifecycle", Severity: "high", MinCount: 1, Keywords: []string{"upgrade stalled", "upgrade stall"}}, + ClassDataDiskPressure: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"disk full", "disk pressure", "no space left"}}, + ClassGitHubAppTokenMintFailed: {Component: "github-client", Severity: "high", MinCount: 1, Keywords: []string{"token mint failed", "installation token"}}, + ClassGitHubAppNotInstalled: {Component: "github-client", Severity: "medium", MinCount: 1, Keywords: []string{"github app installation id is unset", "github app has no installation"}}, + ClassUnexpectedHub5xx: {Component: "dashboard", Severity: "medium", MinCount: hub5xxStreak, Keywords: []string{"hub 5", "hub returned 5"}}, + ClassUnclassifiedRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear}, + ClassAgentStall: {Component: "agent-runtime", Severity: "medium", MinCount: AgentStallNudgeStreak, Keywords: []string{"agent stall", "stalled after kick", "continue nudge"}}, + ClassAgentCrashLoop: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"crash loop", "crash-loop", "crash looping"}}, + ClassCIPollNudge: {Component: "agent-runtime", Severity: "low", MinCount: 1, Window: ciPollNudgeWindow, Keywords: []string{"ci-poll nudge", "polling ci", "stop-polling nudge"}}, + ClassIssueRequestStuck: {Component: "github-relay", Severity: "medium", MinCount: 1, Keywords: []string{"issue-request stuck", "issue request stuck", "issue-request watcher", "relay request stuck"}}, + ClassCruxRuntimeUnavailable: {Component: "agent-runtime", Severity: "high", MinCount: 1, Keywords: []string{"crux runtime unavailable", "crux server unreachable"}}, + ClassEffectLedgerOpStuck: {Component: "mutation-ledger", Severity: "medium", MinCount: 1, Keywords: []string{"effect ledger stuck", "ledger op stuck", "needs reconciliation", "mutation journal unknown"}}, + ClassTmuxSessionCheckFailed: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"tmux has-session failed"}}, + ClassStartupKickInterrupted: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"startup kick dropped: agent restarted or stopped while waiting"}}, + ClassAgentPaneTimeout: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"prompt timeout — dumping pane", "pane content at timeout", "startup kick dropped: cli never became ready", "startup kick dropped: cli never reached input prompt", "kick undeliverable"}}, + ClassProxyTLSHandshakeFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy client tls handshake failed", "transparent proxy tls handshake failed", "copilot sniff: client tls handshake failed"}}, + ClassAgentDirSetupFailed: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to create per-agent xdg dir", "failed to restore per-agent dir mode", "failed to restore interactive home mode", "failed to tighten per-agent dir mode", "failed to tighten interactive home mode", "failed to restore inference home mode", "failed to tighten inference home mode"}}, + ClassReviewEvidenceWriteFail: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"review evidence: could not write bundle"}}, + ClassFeedbackIssueRefreshFailed: {Component: "dashboard", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"feedback: issue refresh failed"}}, + ClassBeadFactIngestFailed: {Component: "knowledge", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to ingest bead fact"}}, + ClassSelfReportClearedFailed: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"fleet report: cleared comment failed"}}, + ClassGitHubAppRepoNotCovered: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"installation does not cover every configured repo", "state=repo-not-covered"}}, + ClassProxyConnectDialFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: connect dial failed", "copilot sniff: upstream dial failed"}}, + ClassProxyPlainForwardRefused: {Component: "github-proxy", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward refused"}}, + ClassInferenceCallerUnknown: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference request from unidentified caller refused"}}, + ClassInferenceNoRoute: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference no route"}}, + ClassInferenceUpstreamError: {Component: "inference-gateway", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"inference upstream error", "inference upstream failed", "inference translate request failed"}}, + ClassSHAPollCommitMsgUnavail: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"sha poll: commit message fetch non-200"}}, + ClassKickFreshnessWithheld: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"kick withheld: every ref failed the freshness check"}}, + ClassModelsEndpointEmpty: {Component: "dashboard", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"no models found from any endpoint"}}, + ClassAgentLoginRequired: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"login required detected", "watchdog: agent credential failure"}}, + ClassPRRequestHeadUnpushed: {Component: "github-relay", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"head branch is not on the remote"}}, + ClassIOScanClassifierUnavail: {Component: "inference-gateway", Severity: "medium", MinCount: 1, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"ioscan classifier failed closed", "ioscan classifier failed open", "ioscan fail-closed blocked kick", "ioscan fail-closed blocked held-pr coordination kick"}}, + ClassAgentGitHubCapReached: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"agent github api hourly cap reached"}}, + ClassGitHubFetchTransient: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"channel commit date: fetch failed"}}, + ClassLabelProvisioningFailed: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue stage labels: provisioning failed"}}, + ClassAgentCLIRestartLoop: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"agent cli crashed or stuck on consent screen"}}, + ClassCopilotCLIHung: {Component: "agent-runtime", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"copilot hung with no cli prompt", "diagnostic: timed out waiting for copilot error output"}}, + ClassSharedCIRerunForbidden: {Component: "github-client", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"shared-ci rerun forbidden"}}, + ClassIssueClaimEffectFailed: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"issue-claims: github side effect failed"}}, + ClassTranscriptScanSlow: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"transcript activity scan outlived its interval"}}, + ClassForgeUnreachable: {Component: "network", Severity: "medium", MinCount: 1, Keywords: []string{"github host unreachable", "forge unreachable"}}, + ClassHubHeartbeatUnprovisioned: {Component: "agent-runtime", Severity: "medium", MinCount: 1, IdleClear: 3 * time.Hour, LogFed: true, Keywords: []string{"hub heartbeat rejected: hive id is not provisioned"}}, + ClassGitHubAppAlertsPermission: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"scoped token mint with vulnerability_alerts:read failed; retrying without it"}}, + ClassGitHubAppStatusesPermission: {Component: "github-client", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"could not set the prow/lgtm commit status: the github app lacks the statuses: write permission"}}, + ClassProxyPlainForwardFailed: {Component: "github-proxy", Severity: "medium", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"proxy: plain http forward failed"}}, + ClassAgentLifecycleRepeated: {Component: "agent-runtime", Severity: "low", MinCount: unclassifiedRepeatCount, Window: unclassifiedRepeatWindow, IdleClear: unclassifiedIdleClear, LogFed: true, Keywords: []string{"failed to send kick", "agent cli crashed (bare shell detected)", "tmux kill-session before restart failed", "agent tmux session missing", "would restart dead agent"}}, } // LogFedClass returns the LogFed class whose Keyword msg contains, or "". diff --git a/src/pkg/fleetreport/runtime_test.go b/src/pkg/fleetreport/runtime_test.go index fc76ebf96c..008fbc120a 100644 --- a/src/pkg/fleetreport/runtime_test.go +++ b/src/pkg/fleetreport/runtime_test.go @@ -433,6 +433,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassUpgradeStalled, message: "standalone upgrade helper did not finish within 10m0s"}, {class: ClassDataDiskPressure, message: "data disk pressure: disk-full: /data is 97% used (threshold 95%)"}, {class: ClassGitHubAppTokenMintFailed, message: "github app token mint failed: POST /app/installations/123/access_tokens: 401"}, + {class: ClassForgeUnreachable, message: "github host unreachable during token mint: creating installation token: context deadline exceeded"}, {class: ClassUnexpectedHub5xx, message: "hub heartbeat returned HTTP 503"}, {class: ClassUnclassifiedRepeated, message: "weird error from somewhere"}, {class: ClassAgentStall, message: "agent stalled after kick: 3 continue nudges sent without the pane moving"}, @@ -441,6 +442,7 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassIssueRequestStuck, message: "issue-request stuck: issue request failed 3 attempts over 7m"}, {class: ClassTmuxSessionCheckFailed, message: "tmux has-session failed"}, {class: ClassAgentPaneTimeout, message: "pane content at timeout"}, + {class: ClassAgentLifecycleRepeated, message: "agent tmux session missing"}, {class: ClassStartupKickInterrupted, message: "startup kick dropped: agent restarted or stopped while waiting"}, {class: ClassProxyTLSHandshakeFailed, message: "proxy client TLS handshake failed"}, {class: ClassAgentDirSetupFailed, message: "failed to create per-agent XDG dir"}, @@ -452,8 +454,12 @@ func TestEveryRuntimeClassTriggerAndNonTriggerThresholds(t *testing.T) { {class: ClassBeadFactIngestFailed, message: "failed to ingest bead fact"}, {class: ClassSelfReportClearedFailed, message: "fleet report: cleared comment failed"}, {class: ClassGitHubAppRepoNotCovered, message: "github app recheck: installation does not cover every configured repo"}, + {class: ClassGitHubAppAlertsPermission, message: "scoped token mint with vulnerability_alerts:read failed; retrying without it"}, + {class: ClassGitHubAppStatusesPermission, message: "could not set the prow/lgtm commit status: the GitHub App lacks the statuses: write permission"}, + {class: ClassHubHeartbeatUnprovisioned, message: "hub heartbeat rejected: hive id is not provisioned on the hub (403); backing off"}, {class: ClassGitHubAppNotInstalled, message: "github app installation id is unset — per-agent scoped token NOT delivered"}, {class: ClassProxyConnectDialFailed, message: "proxy: CONNECT dial failed"}, + {class: ClassProxyPlainForwardFailed, message: "proxy: plain HTTP forward failed"}, {class: ClassProxyPlainForwardRefused, message: "proxy: plain HTTP forward refused — destination is not globally routable"}, {class: ClassInferenceCallerUnknown, message: "inference request from unidentified caller refused"}, {class: ClassInferenceNoRoute, message: "inference no route"}, @@ -1196,53 +1202,59 @@ func TestUnclassifiedRepeatedHonoursWindowAndIdleClear(t *testing.T) { func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { fed := map[string]string{ "startup kick dropped: agent restarted or stopped while waiting": ClassStartupKickInterrupted, - "tmux has-session failed": ClassTmuxSessionCheckFailed, - "pane content at timeout": ClassAgentPaneTimeout, - "prompt timeout — dumping pane": ClassAgentPaneTimeout, - "startup kick dropped: CLI never reached input prompt": ClassAgentPaneTimeout, - "startup kick dropped: CLI never became ready": ClassAgentPaneTimeout, - "channel commit date: fetch failed — the row will render without a timestamp": ClassGitHubFetchTransient, - "issue stage labels: provisioning failed": ClassLabelProvisioningFailed, - "agent CLI crashed or stuck on consent screen, restarting before kick": ClassAgentCLIRestartLoop, - "copilot hung with no CLI prompt, running diagnostic": ClassCopilotCLIHung, - "diagnostic: timed out waiting for copilot error output, relaunching agent": ClassCopilotCLIHung, - "shared-ci rerun forbidden: the App installation lacks Actions write permission": ClassSharedCIRerunForbidden, - "issue-claims: GitHub side effect failed": ClassIssueClaimEffectFailed, - "SHA poll: commit message fetch non-200": ClassSHAPollCommitMsgUnavail, - "kick withheld: every ref failed the freshness check": ClassKickFreshnessWithheld, - "no models found from any endpoint": ClassModelsEndpointEmpty, - "login required detected": ClassAgentLoginRequired, - "pr-request watcher: head branch is not on the remote": ClassPRRequestHeadUnpushed, - "ioscan classifier failed closed; text withheld": ClassIOScanClassifierUnavail, - "watchdog: agent credential failure": ClassAgentLoginRequired, - "ioscan classifier failed open; text passes on unclassified": ClassIOScanClassifierUnavail, - "ioscan fail-closed blocked kick": ClassIOScanClassifierUnavail, - "ioscan fail-closed blocked held-PR coordination kick": ClassIOScanClassifierUnavail, - "proxy client TLS handshake failed": ClassProxyTLSHandshakeFailed, - "transparent proxy TLS handshake failed": ClassProxyTLSHandshakeFailed, - "failed to restore per-agent dir mode": ClassAgentDirSetupFailed, - "failed to restore interactive home mode": ClassAgentDirSetupFailed, - "failed to tighten per-agent dir mode": ClassAgentDirSetupFailed, - "failed to tighten interactive home mode": ClassAgentDirSetupFailed, - "failed to create per-agent XDG dir": ClassAgentDirSetupFailed, - "review evidence: could not write bundle": ClassReviewEvidenceWriteFail, - "feedback: issue refresh failed": ClassFeedbackIssueRefreshFailed, - "failed to restore inference home mode": ClassAgentDirSetupFailed, - "failed to tighten inference home mode": ClassAgentDirSetupFailed, - "failed to ingest bead fact": ClassBeadFactIngestFailed, - "fleet report: cleared comment failed": ClassSelfReportClearedFailed, - "copilot sniff: client TLS handshake failed": ClassProxyTLSHandshakeFailed, - "github app recheck: installation does not cover every configured repo": ClassGitHubAppRepoNotCovered, - "proxy: plain HTTP forward refused — destination is not globally routable": ClassProxyPlainForwardRefused, - "proxy: CONNECT dial failed": ClassProxyConnectDialFailed, - "copilot sniff: upstream dial failed": ClassProxyConnectDialFailed, - "inference no route": ClassInferenceNoRoute, - "inference upstream error": ClassInferenceUpstreamError, - "copilot transcript activity scan outlived its interval; agent liveness reads may lag": ClassTranscriptScanSlow, - "inference upstream failed": ClassInferenceUpstreamError, - "inference translate request failed": ClassInferenceUpstreamError, - "kick undeliverable; busy-no-activity restart": ClassAgentPaneTimeout, - "agent GitHub API hourly cap reached": ClassAgentGitHubCapReached, + "tmux has-session failed": ClassTmuxSessionCheckFailed, + "pane content at timeout": ClassAgentPaneTimeout, + "prompt timeout — dumping pane": ClassAgentPaneTimeout, + "startup kick dropped: CLI never reached input prompt": ClassAgentPaneTimeout, + "startup kick dropped: CLI never became ready": ClassAgentPaneTimeout, + "channel commit date: fetch failed — the row will render without a timestamp": ClassGitHubFetchTransient, + "issue stage labels: provisioning failed": ClassLabelProvisioningFailed, + "agent CLI crashed or stuck on consent screen, restarting before kick": ClassAgentCLIRestartLoop, + "copilot hung with no CLI prompt, running diagnostic": ClassCopilotCLIHung, + "diagnostic: timed out waiting for copilot error output, relaunching agent": ClassCopilotCLIHung, + "shared-ci rerun forbidden: the App installation lacks Actions write permission": ClassSharedCIRerunForbidden, + "issue-claims: GitHub side effect failed": ClassIssueClaimEffectFailed, + "SHA poll: commit message fetch non-200": ClassSHAPollCommitMsgUnavail, + "kick withheld: every ref failed the freshness check": ClassKickFreshnessWithheld, + "no models found from any endpoint": ClassModelsEndpointEmpty, + "login required detected": ClassAgentLoginRequired, + "pr-request watcher: head branch is not on the remote": ClassPRRequestHeadUnpushed, + "ioscan classifier failed closed; text withheld": ClassIOScanClassifierUnavail, + "watchdog: agent credential failure": ClassAgentLoginRequired, + "ioscan classifier failed open; text passes on unclassified": ClassIOScanClassifierUnavail, + "ioscan fail-closed blocked kick": ClassIOScanClassifierUnavail, + "ioscan fail-closed blocked held-PR coordination kick": ClassIOScanClassifierUnavail, + "proxy client TLS handshake failed": ClassProxyTLSHandshakeFailed, + "transparent proxy TLS handshake failed": ClassProxyTLSHandshakeFailed, + "failed to restore per-agent dir mode": ClassAgentDirSetupFailed, + "failed to restore interactive home mode": ClassAgentDirSetupFailed, + "failed to tighten per-agent dir mode": ClassAgentDirSetupFailed, + "failed to tighten interactive home mode": ClassAgentDirSetupFailed, + "failed to create per-agent XDG dir": ClassAgentDirSetupFailed, + "review evidence: could not write bundle": ClassReviewEvidenceWriteFail, + "feedback: issue refresh failed": ClassFeedbackIssueRefreshFailed, + "failed to restore inference home mode": ClassAgentDirSetupFailed, + "failed to tighten inference home mode": ClassAgentDirSetupFailed, + "failed to ingest bead fact": ClassBeadFactIngestFailed, + "fleet report: cleared comment failed": ClassSelfReportClearedFailed, + "copilot sniff: client TLS handshake failed": ClassProxyTLSHandshakeFailed, + "github app recheck: installation does not cover every configured repo": ClassGitHubAppRepoNotCovered, + "proxy: plain HTTP forward refused — destination is not globally routable": ClassProxyPlainForwardRefused, + "proxy: CONNECT dial failed": ClassProxyConnectDialFailed, + "copilot sniff: upstream dial failed": ClassProxyConnectDialFailed, + "inference no route": ClassInferenceNoRoute, + "inference upstream error": ClassInferenceUpstreamError, + "copilot transcript activity scan outlived its interval; agent liveness reads may lag": ClassTranscriptScanSlow, + "inference upstream failed": ClassInferenceUpstreamError, + "inference translate request failed": ClassInferenceUpstreamError, + "kick undeliverable; busy-no-activity restart": ClassAgentPaneTimeout, + "agent GitHub API hourly cap reached": ClassAgentGitHubCapReached, + "failed to send kick": ClassAgentLifecycleRepeated, + "agent CLI crashed (bare shell detected)": ClassAgentLifecycleRepeated, + "tmux kill-session before restart failed": ClassAgentLifecycleRepeated, + "agent tmux session missing": ClassAgentLifecycleRepeated, + "scoped token mint with vulnerability_alerts:read failed; retrying without it": ClassGitHubAppAlertsPermission, + "could not set the prow/lgtm commit status: the GitHub App lacks the statuses: write permission": ClassGitHubAppStatusesPermission, } for msg, want := range fed { if got := LogFedClass(msg); got != want { @@ -1252,6 +1264,11 @@ func TestRecurringRuntimeMessagesAreClassified(t *testing.T) { t.Errorf("%q not matched by an explicit class", msg) } } + // The neutral prow/lgtm line (5xx, 422, transport) must not file as the + // operator-only permission class; only the 403 wording names it. + if got := LogFedClass("could not set the prow/lgtm commit status"); got == ClassGitHubAppStatusesPermission { + t.Errorf("neutral prow/lgtm status failure classified as %q", got) + } for _, m := range []string{ "github REST 404 cached; suppressing repeated fetches", "copilot token validation rate-limited by GitHub; relaunching after cooldown", diff --git a/src/pkg/fleetreport/slogfeed/slogfeed.go b/src/pkg/fleetreport/slogfeed/slogfeed.go index a5ee890b10..5a9507e578 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed.go @@ -14,6 +14,7 @@ package slogfeed import ( "context" + "errors" "log/slog" "strings" @@ -114,7 +115,7 @@ func Key(msg string) string { } func (h *Handler) feed(r slog.Record) { - if expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) { + if canceledError(r, h.attrs) || expectedKickHold(r) || rateLimitedError(r) || sentinelPacingSkip(r) || upstreamUnavailableError(r) || failClosedStatus500(r) || hubHeartbeatUnavailable(r) || hubHeartbeatCachedFallback(r) || reviewCommentCapped(r) || inferenceBlockedStall(r) || automergeMergeGuardSkip(r) || reviewRetryServerError(r) || stageDampenerSuppression(r) || signedCommitOwnershipSkip(r) { return } class := fleetreport.ClassUnclassifiedRepeated @@ -135,6 +136,38 @@ func (h *Handler) feed(r slog.Record) { h.observe(class, key, r.Message, diagnostics(r, h.attrs)) } +// signedCommitOwnershipSkip is the reconciler's deliberate refusal to rewrite +// a foreign or merge commit. The warning and PR remedy remain visible; repeated +// skips across PRs are not a runtime defect. +func signedCommitOwnershipSkip(r slog.Record) bool { + return r.Level == slog.LevelWarn && strings.TrimSpace(r.Message) == "signed-commit reconciler: skipped; the PR keeps an unsigned commit" +} + +// canceledError applies the existing cancellation exclusion to structured +// errors too. An accompanying different error must remain reportable. +func canceledError(r slog.Record, bound []slog.Attr) bool { + found, other := false, false + check := func(a slog.Attr) bool { + if a.Key == "error" { + value := a.Value.Resolve() + err, ok := value.Any().(error) + if ok && errors.Is(err, context.Canceled) { + found = true + } else if text := value.String(); text == context.Canceled.Error() || strings.HasSuffix(text, ": "+context.Canceled.Error()) { + found = true + } else { + other = true + } + } + return true + } + for _, a := range bound { + check(a) + } + r.Attrs(check) + return found && !other +} + // expectedKickHold reports whether a "failed to send kick" record carries the // deliberate post-restart kick cooldown refusal (#7363). The hold is the // breaker working as designed, not a repeated runtime defect (#14422); other diff --git a/src/pkg/fleetreport/slogfeed/slogfeed_test.go b/src/pkg/fleetreport/slogfeed/slogfeed_test.go index a869cc41a3..c3cfd32828 100644 --- a/src/pkg/fleetreport/slogfeed/slogfeed_test.go +++ b/src/pkg/fleetreport/slogfeed/slogfeed_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "errors" + "fmt" "log/slog" "net/url" "strings" @@ -64,6 +65,89 @@ func TestHandlerSkipsBenignAndExplicitClasses(t *testing.T) { } } +func TestSignedCommitOwnershipSkipStaysInLogsWithoutFleetReport(t *testing.T) { + logger, seen, out := newFeed(t) + const msg = "signed-commit reconciler: skipped; the PR keeps an unsigned commit" + for range 6 { + logger.Warn(msg, "repo", "o/r", "pr", 1403, "reason", "commit is authored by another account") + } + if len(*seen) != 0 { + t.Fatalf("expected ownership skips produced observations: %+v", *seen) + } + if n := strings.Count(out.String(), msg); n != 6 || !strings.Contains(out.String(), "another account") { + t.Fatalf("local logs lost the warning or its reason: %s", out.String()) + } + logger.Warn("signed-commit reconciler: listing PR commits failed", "err", "unexpected failure") + logger.Error(msg, "reason", "unexpected failure") + if len(*seen) != 2 || (*seen)[0].class != fleetreport.ClassUnclassifiedRepeated || (*seen)[1].class != fleetreport.ClassUnclassifiedRepeated { + t.Fatalf("other failures must remain reportable: %+v", *seen) + } +} + +func TestCanceledStageDigestStaysLocal(t *testing.T) { + logger, seen, out := newFeed(t) + const msg = "stage labels: check digest failed" + for range 6 { + logger.Warn(msg, "repo", "utah", "pr", 1194, "error", context.Canceled) + } + logger.Warn(msg, "error", errors.New("GET checks: context canceled")) + if len(*seen) != 0 { + t.Fatalf("canceled reads produced fleet observations: %+v", *seen) + } + if n := strings.Count(out.String(), "\n"); n != 7 { + t.Fatalf("local log records = %d, want 7", n) + } + logger.Warn(msg, "error", errors.New("context deadline exceeded")) + logger.Warn(msg, "error", errors.New("permission denied")) + logger.Warn("another failure", "error", context.Canceled) + if len(*seen) != 2 { + t.Fatalf("unexpected errors must remain reportable: %+v", *seen) + } +} + +func TestCanceledAdoptionReadStaysInLog(t *testing.T) { + logger, seen, out := newFeed(t) + const msg = "stage labels: could not read maintainer_can_modify; keeping in-place adoption this sweep" + for range 8 { + logger.Warn(msg, "repo", "server", "pr", 649, "error", &url.Error{Op: "GET", URL: "https://api.github.com", Err: context.Canceled}) + } + logger.Warn(msg, "error", "context canceled") + if len(*seen) != 0 || strings.Count(out.String(), "\n") != 9 { + t.Fatalf("canceled reads: observations=%+v, log=%s", *seen, out.String()) + } + logger.Warn(msg, "error", errors.New("permission denied")) + logger.Warn(msg, "error", context.DeadlineExceeded) + logger.Warn("another operation failed", "error", context.Canceled) + if len(*seen) != 2 { + t.Fatalf("other failures must remain reportable: %+v", *seen) + } +} + +func TestStructuredCancellationDoesNotCreateFleetReport(t *testing.T) { + logger, seen, out := newFeed(t) + const msg = "stage labels reconcile failed" + for range 10 { + logger.Warn(msg, "error", context.Canceled, "items", 6, "writes", 0) + } + logger.With("error", context.Canceled).Warn(msg) + logger.Warn("stage labels: check digest failed", "error", fmt.Errorf("GET checks: %w", context.Canceled)) + logger.Warn("stage labels: check digest failed", "error", "GET checks: context canceled") + logger.With("error", fmt.Errorf("listing PRs: %w", context.Canceled)).Warn(msg, "error", context.Canceled) + logger.Warn(msg, "error", &url.Error{Op: "GET", URL: "https://api.github.com", Err: context.Canceled}) + if len(*seen) != 0 || strings.Count(out.String(), "\n") != 15 { + t.Fatalf("observed=%+v logs=%s", *seen, out.String()) + } + logger.Warn(msg, "error", context.DeadlineExceeded) + logger.Warn(msg, "error", "permission denied") + logger.With("error", context.Canceled).Warn(msg, "error", "permission denied") + logger.With("error", fmt.Errorf("listing PRs: %w", context.Canceled)).Warn(msg, "error", "permission denied") + logger.Warn(msg, "error", "context canceled: permission denied") + logger.Warn(msg, "error", fmt.Errorf("request: %w", context.DeadlineExceeded)) + if len(*seen) != 6 { + t.Fatalf("other failures must remain reportable: %+v", *seen) + } +} + // #16510: polling an exhausted rebase episode must keep warning locally // without turning the intentional handoff into a repeated runtime defect. func TestRebaseAttemptsExhaustedLogsWithoutFleetReport(t *testing.T) { @@ -359,6 +443,26 @@ func TestRequiredChecksOverrideMismatchLogsWithoutFleetReport(t *testing.T) { } } +func TestMissingAlertsPermissionReportsOperatorCondition(t *testing.T) { + now := time.Date(2026, 11, 12, 15, 40, 0, 0, time.UTC) + tr := fleetreport.NewRuntimeTracker(func() time.Time { return now }) + var out bytes.Buffer + logger := slog.New(NewWithObserver(slog.NewJSONHandler(&out, nil), tr.Observe)) + const msg = "scoped token mint with vulnerability_alerts:read failed; retrying without it — the sec-check Dependabot alerts call will 403 until the App grants Dependabot alerts read-only and the installation re-accepts it; see docs/github-app-setup.md" + for range 5 { + logger.Warn(msg, "tier", "sec-check", "error", "permissions requested are not granted") + now = now.Add(time.Minute) + } + ev := tr.Evidence() + if len(ev) != 1 || ev[0].Class != fleetreport.ClassGitHubAppAlertsPermission || ev[0].Count != 5 { + t.Fatalf("missing permission evidence = %+v", ev) + } + now = now.Add(21 * time.Minute) + if ev := tr.Evidence(); len(ev) != 0 { + t.Fatalf("quiet condition did not clear: %+v", ev) + } +} + func TestFeedLogFedPath(t *testing.T) { tests := []struct { name string @@ -512,7 +616,7 @@ func TestPostRestartKickHoldLogsWithoutFleetReport(t *testing.T) { } // Other kick send failures must remain reportable. logger.Warn("failed to send kick", "agent", "reviewer-2", "error", "tmux send-keys exit 1") - if len(*seen) != 1 || (*seen)[0].class != fleetreport.ClassUnclassifiedRepeated { + if len(*seen) != 1 || (*seen)[0].class != fleetreport.ClassAgentLifecycleRepeated { t.Fatalf("other kick failures must remain reportable: %+v", *seen) } } diff --git a/src/pkg/gfi/gfi.go b/src/pkg/gfi/gfi.go index 1a5c77ef35..a75440a015 100644 --- a/src/pkg/gfi/gfi.go +++ b/src/pkg/gfi/gfi.go @@ -447,6 +447,17 @@ type Forge interface { UpsertIssueMarkerComment(ctx context.Context, repo string, number int, marker, body string, create bool) (MarkerComment, error) } +// ArchivedWriteTracker remembers GitHub's explicit archived-repository write +// refusals. A nil error asks whether the repository was already recorded. +type ArchivedWriteTracker interface { + ArchivedWrite(repo string, err error) bool +} + +func archivedWrite(forge Forge, repo string, err error) bool { + tracker, ok := forge.(ArchivedWriteTracker) + return ok && tracker.ArchivedWrite(repo, err) +} + // LabelReader is the optional half of Forge the sweep uses to re-read an // issue's labels immediately before advertising it. Candidates come from // the last enumeration, which can be many minutes old; hivecommons/hive @@ -653,6 +664,10 @@ func (s *Sweeper) Sweep(ctx context.Context, forge Forge, candidates []Candidate labelledPerRepo := map[string]int{} for _, c := range candidates { res.Scanned++ + if archivedWrite(forge, c.Repo, nil) { + res.Skipped["archived-repo"]++ + continue + } if lowerSet(c.Labels)[Label] { labelledPerRepo[c.Repo]++ } @@ -688,6 +703,11 @@ func (s *Sweeper) Sweep(ctx context.Context, forge Forge, candidates []Candidate if d.AlreadyLabelled && Withdrawable(d.Skip) { if _, released := s.releasedUntil(c.Key(), p.ReselectCooldown); !released { if err := s.Withdraw(ctx, forge, c, d.Skip, opts); err != nil { + if archivedWrite(forge, c.Repo, err) { + logger.Info("gfi: repository is archived and read-only; skipping until restart", "repo", c.Repo, "error", err) + res.Skipped["archived-repo"]++ + continue + } res.Errors++ } else { res.Withdrawn = append(res.Withdrawn, c.Key()) @@ -714,6 +734,10 @@ func (s *Sweeper) Sweep(ctx context.Context, forge Forge, candidates []Candidate }) picked := 0 for _, sc := range list { + if archivedWrite(forge, repo, nil) { + res.Skipped["archived-repo"]++ + continue + } if picked >= p.MaxPerSweep { res.Skipped["sweep-cap"]++ logger.Debug("gfi skip", "issue", sc.c.Key(), "reason", "sweep-cap") @@ -744,6 +768,11 @@ func (s *Sweeper) Sweep(ctx context.Context, forge Forge, candidates []Candidate } added, err := s.apply(ctx, forge, sc.c, body) if err != nil { + if archivedWrite(forge, repo, err) { + logger.Info("gfi: repository is archived and read-only; skipping until restart", "repo", repo, "error", err) + res.Skipped["archived-repo"]++ + continue + } if errors.Is(err, errSuperseded) { res.Skipped["previously-released"]++ logger.Debug("gfi skip", "issue", sc.c.Key(), "reason", "previously-released") diff --git a/src/pkg/gfi/gfi_test.go b/src/pkg/gfi/gfi_test.go index e69e31915b..c13ed34a3c 100644 --- a/src/pkg/gfi/gfi_test.go +++ b/src/pkg/gfi/gfi_test.go @@ -327,6 +327,54 @@ func quietOpts() SweepOptions { return SweepOptions{Links: Links{Origin: "https://hive.example", HiveID: "acme-hive"}, Now: now, Logger: slog.New(slog.DiscardHandler)} } +type archivedForge struct { + *fakeForge + archived bool + attempts int +} + +var archivedErr = errors.New("archived repository") + +func (f *archivedForge) ArchivedWrite(repo string, err error) bool { + if repo != "acme/archived" { + return false + } + if errors.Is(err, archivedErr) { + f.archived = true + } + return f.archived +} + +func (f *archivedForge) UpsertIssueMarkerComment(ctx context.Context, repo string, n int, marker, body string, create bool) (MarkerComment, error) { + if repo == "acme/archived" && create { + f.attempts++ + return MarkerComment{}, archivedErr + } + return f.fakeForge.UpsertIssueMarkerComment(ctx, repo, n, marker, body, create) +} + +func TestSweepSkipsArchivedRepoAfterRefusal(t *testing.T) { + forge := &archivedForge{fakeForge: newFakeForge()} + s := NewSweeper() + a := good() + a.Repo = "acme/archived" + b := a + b.Number++ + active := good() + candidates := []Candidate{a, b, active} + first := s.Sweep(context.Background(), forge, candidates, quietOpts()) + second := s.Sweep(context.Background(), forge, candidates, quietOpts()) + if forge.attempts != 1 || first.Errors != 0 || second.Errors != 0 { + t.Fatalf("attempts=%d first=%+v second=%+v", forge.attempts, first, second) + } + if first.Skipped["archived-repo"] != 2 || second.Skipped["archived-repo"] != 2 { + t.Fatalf("archived skips: first=%+v second=%+v", first, second) + } + if len(first.Selected) != 1 || first.Selected[0] != active.Key() { + t.Fatalf("active repo not selected: %+v", first) + } +} + func TestSweepSelectsLabelsAndCommentsIdempotently(t *testing.T) { forge := newFakeForge() s := NewSweeper() diff --git a/src/pkg/github/advisory.go b/src/pkg/github/advisory.go index cb62d2e709..6ddf8f0930 100644 --- a/src/pkg/github/advisory.go +++ b/src/pkg/github/advisory.go @@ -302,6 +302,9 @@ func (c *Client) EnsureAdvisoryIssue(ctx context.Context, repo string) (int, err var apiErr error issue, _, apiErr = c.client.Issues.Create(ctx, owner, repo, req) if apiErr != nil { + if isDefiniteIssueRefusal(apiErr) { + return effects.Result{}, effects.NotApplied(apiErr) + } return effects.Result{}, apiErr } return effects.Result{Provenance: issue.GetHTMLURL()}, nil diff --git a/src/pkg/github/advisory_retry_test.go b/src/pkg/github/advisory_retry_test.go new file mode 100644 index 0000000000..85e52ada7e --- /dev/null +++ b/src/pkg/github/advisory_retry_test.go @@ -0,0 +1,81 @@ +package github + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/effects" +) + +// A rejected advisory create must not fence the same claim after access is +// restored; an ambiguous failure must still fence it to prevent duplicates. +func TestEnsureAdvisoryIssue_CreateFailureJournal(t *testing.T) { + for _, tc := range []struct { + name string + status int + definite bool + }{ + {"unauthorized", http.StatusUnauthorized, true}, + {"forbidden", http.StatusForbidden, true}, + {"private repository not selected", http.StatusNotFound, true}, + {"issues disabled", http.StatusGone, true}, + {"validation", http.StatusUnprocessableEntity, true}, + {"server error", http.StatusBadGateway, false}, + } { + t.Run(tc.name, func(t *testing.T) { + var creates int + mux := http.NewServeMux() + mux.HandleFunc("/search/issues", func(w http.ResponseWriter, r *http.Request) { + json.NewEncoder(w).Encode(map[string]any{"total_count": 0, "items": []any{}}) + }) + mux.HandleFunc("/repos/o/r/issues", func(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + json.NewEncoder(w).Encode([]any{}) + return + } + creates++ + if creates == 1 { + w.WriteHeader(tc.status) + fmt.Fprint(w, `{"message":"create failed"}`) + return + } + w.WriteHeader(http.StatusCreated) + fmt.Fprint(w, `{"number":7,"html_url":"https://github.example/o/r/issues/7"}`) + }) + mux.HandleFunc("/repos/o/r/labels", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusCreated) + fmt.Fprint(w, `{"name":"hive/advisory"}`) + }) + mux.HandleFunc("/repos/o/r", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"full_name":"o/r","has_issues":true}`) + }) + server := httptest.NewServer(mux) + defer server.Close() + + c := newTestClient(t, server, "o", []string{"r"}) + c.SetMutationBoundary(replayTestBoundary(t)) + previousNow := issueCreateNow + issueCreateNow = func() time.Time { return time.Date(2026, 10, 13, 12, 0, 0, 0, time.UTC) } + t.Cleanup(func() { issueCreateNow = previousNow }) + + _, err := c.EnsureAdvisoryIssue(context.Background(), "r") + if err == nil || errors.Is(err, effects.ErrNotApplied) != tc.definite || githubStatusOf(err) != tc.status { + t.Fatalf("first create: err=%v, want status %d and definite=%v", err, tc.status, tc.definite) + } + number, err := c.EnsureAdvisoryIssue(context.Background(), "r") + if tc.definite { + if err != nil || number != 7 || creates != 2 { + t.Fatalf("retry after refusal: number=%d err=%v POSTs=%d, want 7, nil, 2", number, err, creates) + } + } else if !errors.Is(err, effects.ErrNeedsReconciliation) || creates != 1 { + t.Fatalf("retry after ambiguous failure: number=%d err=%v POSTs=%d, want reconciliation and 1 POST", number, err, creates) + } + }) + } +} diff --git a/src/pkg/github/app_coverage.go b/src/pkg/github/app_coverage.go index 0b60a738e8..c7107250fe 100644 --- a/src/pkg/github/app_coverage.go +++ b/src/pkg/github/app_coverage.go @@ -50,6 +50,54 @@ func (a *AppAuth) InstallationCoverage(ctx context.Context) (InstallationCoverag return cov, fmt.Errorf("minting installation token: %w", err) } + return a.installationCoverageWithToken(ctx, token) +} + +// InstallationCoverageForID lists repositories for a discovered installation +// without changing the installation or cached token used by the hive. +func (a *AppAuth) InstallationCoverageForID(ctx context.Context, installationID int64) (InstallationCoverage, error) { + if a == nil || !a.HasKey() { + return InstallationCoverage{}, fmt.Errorf("no GitHub App key loaded") + } + if installationID <= 0 { + return InstallationCoverage{}, fmt.Errorf("invalid installation ID") + } + token, _, err := a.mintInstallationTokenWithOptions(ctx, installationID, nil) + if err != nil { + return InstallationCoverage{}, fmt.Errorf("minting installation token: %w", err) + } + return a.installationCoverageWithToken(ctx, token) +} + +// RepositoryCoveredByInstallation checks one repository with the App JWT. +// Unlike a repository GET with an installation token, this endpoint identifies +// the actual installation and cannot mistake public read access for selection. +// A 404 means the App has no installation for the repository; other failures +// and an unexpected installation are inconclusive. +func (a *AppAuth) RepositoryCoveredByInstallation(ctx context.Context, installationID int64, owner, repo string) (bool, error) { + if a == nil || !a.HasKey() || installationID <= 0 || owner == "" || repo == "" { + return false, fmt.Errorf("invalid repository installation probe") + } + token, err := a.generateJWT() + if err != nil { + return false, fmt.Errorf("generating JWT: %w", err) + } + inst, resp, err := newJWTClient(token, a.apiURL).Apps.FindRepositoryInstallation(ctx, owner, repo) + if err != nil { + if resp != nil && resp.StatusCode == 404 { + return false, nil + } + return false, fmt.Errorf("finding repository installation: %w", err) + } + if inst == nil || inst.GetID() != installationID { + return false, fmt.Errorf("repository installation does not match discovered installation %d", installationID) + } + return true, nil +} + +func (a *AppAuth) installationCoverageWithToken(ctx context.Context, token string) (InstallationCoverage, error) { + cov := InstallationCoverage{Repos: map[string]struct{}{}} + client := newTokenClient(token, a.apiURL) opts := &gh.ListOptions{PerPage: coveragePageSize} diff --git a/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go b/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go index f1ebb2f69c..04e9252929 100644 --- a/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go +++ b/src/pkg/github/automerge/automerge_contributor_prow_oktotest.go @@ -68,11 +68,14 @@ func (c *Engine) requestProwPresubmitsForFork(ctx context.Context, displayRepo, } if state.rejected { c.markWorkflowApprovalUntrusted(owner, repo, number) - if !state.rejectionNoted { - c.postWorkflowApprovalUntrustedNote(ctx, displayRepo, owner, repo, number, headSHA, prowPresubmitsRejectedNote(strings.TrimSpace(c.transport.AppBotLogin()))) + args := []any{"repo", displayRepo, "pr", number, "sha", headSHA, "skipped", hgithub.PRApprovalCommandsSkippedProw} + const msg = "contributor-lane ok-to-test not posted again: Prow rejected the App's command on this fork PR; a trusted user must post it" + if state.rejectionNoted { + c.debug(msg, args...) + return } - c.warn("contributor-lane ok-to-test not posted again: Prow rejected the App's command on this fork PR; a trusted user must post it", - "repo", displayRepo, "pr", number, "sha", headSHA, "skipped", hgithub.PRApprovalCommandsSkippedProw) + c.postWorkflowApprovalUntrustedNote(ctx, displayRepo, owner, repo, number, headSHA, prowPresubmitsRejectedNote(strings.TrimSpace(c.transport.AppBotLogin()))) + c.warn(msg, args...) return } if state.postedForHead || !(hgithub.ProwManagedLabels(labels) || state.prowManaged) { diff --git a/src/pkg/github/automerge/automerge_contributor_prow_oktotest_test.go b/src/pkg/github/automerge/automerge_contributor_prow_oktotest_test.go index b884c6cfd8..2a985bd06a 100644 --- a/src/pkg/github/automerge/automerge_contributor_prow_oktotest_test.go +++ b/src/pkg/github/automerge/automerge_contributor_prow_oktotest_test.go @@ -7,7 +7,9 @@ package automerge // that shape, never otherwise, and stops for good once Prow rejects the App. import ( + "bytes" "context" + "log/slog" "strings" "testing" "time" @@ -181,3 +183,28 @@ func TestContributorLaneRelayEvaluationDoesNotPostProwOkToTest(t *testing.T) { t.Fatalf("comments = %d %q, want none from a relay evaluation", fx.comments, fx.allComments) } } + +// A rejection the PR thread already records is a handled state: a restarted +// hive must not log it at WARN again. +func TestContributorLaneProwRejectionAlreadyNotedLogsBelowWarn(t *testing.T) { + fx := prowOkToTestFixture() + fx.prComments = []map[string]any{ + appComment(2, workflowApprovalCommandBody("oldhead")), + prowComment(3, "@"+testHiveAppBotLogin+": Cannot trigger testing until a trusted user reviews the PR and leaves an `/ok-to-test` message."), + } + api := newContributorLaneAPI(t, &fx) + defer api.Close() + policy := defaultContributorLanePolicy() + clock := &contributorLaneClock{at: time.Date(2026, 11, 9, 12, 0, 0, 0, time.UTC)} + var logs bytes.Buffer + for run := 0; run < 2; run++ { + engine := newContributorLaneEngine(api.URL, policy, clock, Options{ContributorSoak: soakedTracker(clock)}) + engine.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})) + if _, _, err := engine.trySweepContributorLanePR(context.Background(), "widget", "acme", "widget", 7, policy); err != nil { + t.Fatalf("run %d err = %v", run, err) + } + } + if got := strings.Count(logs.String(), "ok-to-test not posted again"); got != 1 { + t.Fatalf("WARN lines = %d in %q, want 1 (only the run that left the note)", got, logs.String()) + } +} diff --git a/src/pkg/github/automerge/automerge_held_runs.go b/src/pkg/github/automerge/automerge_held_runs.go index f5ce42fc8e..94152ede07 100644 --- a/src/pkg/github/automerge/automerge_held_runs.go +++ b/src/pkg/github/automerge/automerge_held_runs.go @@ -127,14 +127,20 @@ const ( // is still treated as "GitHub has not scheduled them yet". heldRunNeverStartedGrace = 30 * time.Minute workflowRunsPageSize = 100 + // workflowRunStatusQueued is the status of a run GitHub has not started. + workflowRunStatusQueued = "queued" + // orphanedRunWarnKey keys the once-per-head orphaned-run warning in + // terminalHoldLogged. + orphanedRunWarnKey = "orphaned-queued-runs:" ) // resolveMissingRequiredCheck refines a `required-check-missing:<ctx>` skip // for a trusted-lane PR. It returns the original reason when nothing is known, // a workflow-approval-required reason when GitHub is waiting for approval and // Hive has posted `/ok-to-test`, or a never-started/never-ran reason when -// polling cannot progress. It never approves a run with the App token, and it -// never posts a command to Dependabot, which refuses commands from apps. +// polling cannot progress. Runs GitHub orphaned in `queued` with zero jobs +// count as never started (#17775). It never approves a run with the App token, +// and it never posts a command to Dependabot, which refuses commands from apps. func (c *Engine) resolveMissingRequiredCheck(ctx context.Context, displayRepo, owner, repo string, pr *gh.PullRequest, number int, headSHA, reason string, headPushedAt time.Time) string { if c == nil || c.gh == nil || !strings.HasPrefix(reason, requiredCheckMissingPrefix) { return reason @@ -154,6 +160,14 @@ func (c *Engine) resolveMissingRequiredCheck(ctx context.Context, displayRepo, o } held := heldSameRepoWorkflowRuns(runs.WorkflowRuns, owner+"/"+repo) if len(held) == 0 { + startedRuns := len(runs.WorkflowRuns) + if orphaned := c.orphanedQueuedWorkflowRuns(ctx, owner, repo, runs.WorkflowRuns); orphaned > 0 && orphaned == startedRuns { + startedRuns = 0 + if c.markTerminalHoldLogged(owner, repo, number, headSHA, orphanedRunWarnKey+missing) { + c.warn("automerge workflow runs for head are orphaned in queued with no jobs; treating them as never started", + "repo", displayRepo, "pr", number, "sha", headSHA, "orphaned", orphaned, "context", missing) + } + } if c.botTakeoverAttempted(owner, repo, number, headSHA) { return requiredCheckNeverRanPrefix + missing } @@ -166,10 +180,10 @@ func (c *Engine) resolveMissingRequiredCheck(ctx context.Context, displayRepo, o "repo", displayRepo, "pr", number, "sha", headSHA, "context", missing) return requiredCheckNeverRanAutofixPrefix + missing } - if c.botTakeoverEligible(pr, len(runs.WorkflowRuns), headPushedAt) { + if c.botTakeoverEligible(pr, startedRuns, headPushedAt) { return c.takeOverBotPR(ctx, displayRepo, owner, repo, pr, number, headSHA, missing) } - if len(runs.WorkflowRuns) == 0 && headOlderThan(c.now, headPushedAt, heldRunNeverStartedGrace) { + if startedRuns == 0 && headOlderThan(c.now, headPushedAt, heldRunNeverStartedGrace) { return requiredCheckNeverStartedPrefix + missing } return reason @@ -600,6 +614,32 @@ func heldSameRepoWorkflowRuns(runs []*gh.WorkflowRun, fullName string) []*gh.Wor return held } +// orphanedQueuedWorkflowRuns counts same-repo runs GitHub created but left in +// `queued` past heldRunNeverStartedGrace without materializing a single job. +// Such runs never progress, so they are no evidence CI is coming (#17775). +func (c *Engine) orphanedQueuedWorkflowRuns(ctx context.Context, owner, repo string, runs []*gh.WorkflowRun) int { + orphaned := 0 + for _, run := range runs { + if run == nil || !strings.EqualFold(run.GetStatus(), workflowRunStatusQueued) { + continue + } + if run.GetHeadRepository() == nil || !strings.EqualFold(run.GetHeadRepository().GetFullName(), owner+"/"+repo) { + continue + } + if !headOlderThan(c.now, run.GetCreatedAt().Time, heldRunNeverStartedGrace) { + continue + } + jobs, _, err := c.gh.Actions.ListWorkflowJobs(hgithub.WithRESTCaller(ctx, "hive:automerge_sweep"), owner, repo, run.GetID(), &gh.ListWorkflowJobsOptions{ + ListOptions: gh.ListOptions{PerPage: 1}, + }) + if err != nil || jobs == nil || jobs.GetTotalCount() > 0 || len(jobs.Jobs) > 0 { + continue + } + orphaned++ + } + return orphaned +} + func headOlderThan(now func() time.Time, headPushedAt time.Time, grace time.Duration) bool { if headPushedAt.IsZero() { return false diff --git a/src/pkg/github/automerge/automerge_held_runs_test.go b/src/pkg/github/automerge/automerge_held_runs_test.go index 817defe6cd..98751a08ed 100644 --- a/src/pkg/github/automerge/automerge_held_runs_test.go +++ b/src/pkg/github/automerge/automerge_held_runs_test.go @@ -130,6 +130,61 @@ func TestTrySweepSelfAuthoredPRNamesNeverStartedRequiredCheck(t *testing.T) { } } +// TestTrySweepSelfAuthoredPRTreatsOrphanedQueuedRunsAsNeverStarted pins +// #17775: GitHub can leave a head's runs `queued` with zero jobs forever, and +// such runs must not keep the PR at a bare required-check-missing reason. +func TestTrySweepSelfAuthoredPRTreatsOrphanedQueuedRunsAsNeverStarted(t *testing.T) { + now := time.Date(2026, 11, 11, 22, 10, 0, 0, time.UTC) + old := now.Add(-2 * heldRunNeverStartedGrace).Format(time.RFC3339) + fresh := now.Add(-time.Minute).Format(time.RFC3339) + queuedRun := func(id int, headRepo, createdAt string) map[string]any { + return map[string]any{ + "id": id, "name": "v2 CI", "status": "queued", "created_at": createdAt, + "head_repository": map[string]any{"full_name": headRepo}, + } + } + for _, tc := range []struct { + name string + runs []map[string]any + jobs map[int]int + want string + }{ + {name: "old queued runs with zero jobs", runs: []map[string]any{queuedRun(1, "acme/widget", old), queuedRun(2, "acme/widget", old)}, want: requiredCheckNeverStartedPrefix + "ci/build"}, + {name: "queued run with jobs", runs: []map[string]any{queuedRun(1, "acme/widget", old)}, jobs: map[int]int{1: 1}, want: "required-check-missing:ci/build"}, + {name: "freshly queued run", runs: []map[string]any{queuedRun(1, "acme/widget", fresh)}, want: "required-check-missing:ci/build"}, + {name: "fork head queued run", runs: []map[string]any{queuedRun(1, "mallory/widget", old)}, want: "required-check-missing:ci/build"}, + { + name: "orphaned run beside an in-progress run", + runs: []map[string]any{queuedRun(1, "acme/widget", old), {"id": 2, "name": "Tests", "status": "in_progress", "head_repository": map[string]any{"full_name": "acme/widget"}}}, + want: "required-check-missing:ci/build", + }, + } { + t.Run(tc.name, func(t *testing.T) { + pr := missingRequiredPR(func(p *selfAuthoredPR) { + p.workflowRuns = tc.runs + p.workflowRunJobs = tc.jobs + p.updatedAt = old + }) + var merged []int + api := newSelfAuthoredAutoMergeAPI(t, []selfAuthoredPR{pr}, &merged) + defer api.Close() + c := newAutoMergeSweepClient(api.URL) + c.now = func() time.Time { return now } + + _, reason, err := c.trySweepSelfAuthoredPR(context.Background(), "widget", "acme", "widget", 23, true) + if err != nil { + t.Fatalf("trySweepSelfAuthoredPR returned error: %v", err) + } + if reason != tc.want { + t.Fatalf("reason = %q, want %q", reason, tc.want) + } + if len(merged) != 0 { + t.Fatalf("merged = %v, want no merge", merged) + } + }) + } +} + func TestTerminalRequiredCheckHoldLoggedOncePerHeadAndReason(t *testing.T) { tests := []struct { name string diff --git a/src/pkg/github/automerge/automerge_stage_dco.go b/src/pkg/github/automerge/automerge_stage_dco.go index 5237934be7..a8af6c9969 100644 --- a/src/pkg/github/automerge/automerge_stage_dco.go +++ b/src/pkg/github/automerge/automerge_stage_dco.go @@ -84,7 +84,7 @@ func dcoCommitStatsForHive(commits []*gh.RepositoryCommit, appLogin, prAuthor st } else { stats.unsignedOther++ } - } else if !selfSquash && dcoCommitOwnedByHive(rc, appLogin, isHive) && !messageHasPRAuthorSignoff(msg, prAuthor) { + } else if !selfSquash && dcoCommitOwnedByHive(rc, appLogin, isHive) && !commitHasPRAuthorSignoff(rc, prAuthor) { stats.squashUnsafeHive++ } } @@ -191,6 +191,34 @@ func botSquashAuthor(login string) bool { return strings.HasSuffix(strings.ToLower(strings.TrimSpace(login)), "[bot]") } +// commitHasPRAuthorSignoff mirrors signoff_is_pr_author in +// check-squash-signoff-attribution.sh: a sign-off certifies the PR author +// when it is that account's GitHub noreply, or when GitHub links the commit +// AUTHOR to the PR author and the sign-off is that author's own git email +// (or noreply). The committer is not consulted, so a hive rebase that makes +// the App the committer of the author's own signed commit does not file it +// as the hive's to re-sign (hivecommons/hive#17789). +func commitHasPRAuthorSignoff(commit *gh.RepositoryCommit, prAuthor string) bool { + msg := commit.GetCommit().GetMessage() + if messageHasPRAuthorSignoff(msg, prAuthor) { + return true + } + want := normalizeBotLogin(prAuthor) + if want == "" || normalizeBotLogin(commit.GetAuthor().GetLogin()) != want { + return false + } + authorEmail := strings.ToLower(strings.TrimSpace(commit.GetCommit().GetAuthor().GetEmail())) + for _, line := range strings.Split(msg, "\n") { + if !strings.HasPrefix(strings.ToLower(strings.TrimSpace(line)), "signed-off-by:") { + continue + } + if email := signedOffEmail(line); email != "" && email == authorEmail { + return true + } + } + return false +} + func messageHasPRAuthorSignoff(message, prAuthor string) bool { login := strings.ToLower(strings.TrimSpace(prAuthor)) if login == "" { diff --git a/src/pkg/github/automerge/automerge_stage_labels_test.go b/src/pkg/github/automerge/automerge_stage_labels_test.go index 84dbe0be8a..e7da673f7a 100644 --- a/src/pkg/github/automerge/automerge_stage_labels_test.go +++ b/src/pkg/github/automerge/automerge_stage_labels_test.go @@ -2508,6 +2508,42 @@ func TestDCOCommitStatsForHiveSquashUnsafeCountsHiveIdentities(t *testing.T) { } } +// hivecommons/hive#17789: a hive rebase + force-push makes the App bot the +// committer of the PR author's own commit. A sign-off with the commit +// author's own email, on a commit GitHub links to the PR author, certifies +// the PR author (the squash attribution gate agrees), so it is never +// squash-unsafe; a foreign sign-off under the same committer still is. +func TestDCOCommitStatsForHiveAuthorSignedAfterHiveRebase(t *testing.T) { + rebased := func(authorLogin, authorEmail, signoff string) *gh.RepositoryCommit { + return &gh.RepositoryCommit{ + Author: &gh.User{Login: gh.Ptr(authorLogin)}, + Committer: &gh.User{Login: gh.Ptr(testHiveAppBotLogin)}, + Commit: &gh.Commit{ + Author: &gh.CommitAuthor{Email: gh.Ptr(authorEmail)}, + Message: gh.Ptr("fix: openshell sandbox names\n\nSigned-off-by: " + signoff), + }, + Parents: []*gh.Commit{{}}, + } + } + for _, tc := range []struct { + name string + commit *gh.RepositoryCommit + want int + }{ + {"author's real email", rebased("clubanderson", "andy@clubanderson.com", "Andrew Anderson <andy@clubanderson.com>"), 0}, + {"author email case differs", rebased("ClubAnderson", "Andy@ClubAnderson.com", "Andrew Anderson <andy@clubanderson.com>"), 0}, + {"author's noreply", rebased("clubanderson", "andy@clubanderson.com", "A <123+clubanderson@users.noreply.github.com>"), 0}, + {"someone else's sign-off", rebased("clubanderson", "andy@clubanderson.com", "Other <other@example.com>"), 1}, + {"commit authored by someone else", rebased("collaborator", "andy@clubanderson.com", "Andrew Anderson <andy@clubanderson.com>"), 1}, + {"author login unlinked", rebased("", "andy@clubanderson.com", "Andrew Anderson <andy@clubanderson.com>"), 1}, + } { + stats := dcoCommitStatsForHive([]*gh.RepositoryCommit{tc.commit}, testHiveAppBotLogin, "clubanderson", false, nil) + if stats.unsigned != 0 || stats.squashUnsafeHive != tc.want { + t.Errorf("%s: %+v, want squashUnsafeHive=%d", tc.name, stats, tc.want) + } + } +} + // hivecommons/hive#16701: a split cached before hive identities counted filed // a project.ai_author commit as "other"; stageDCO reads that head's commits // once more so the hive's own commit can be re-signed, then trusts the diff --git a/src/pkg/github/automerge/automerge_sweep_test.go b/src/pkg/github/automerge/automerge_sweep_test.go index c29b239f7f..c4799ce27f 100644 --- a/src/pkg/github/automerge/automerge_sweep_test.go +++ b/src/pkg/github/automerge/automerge_sweep_test.go @@ -1873,6 +1873,9 @@ type selfAuthoredPR struct { updatedAt string // workflowRuns are returned by GET /actions/runs?head_sha=<head>. workflowRuns []map[string]any + // workflowRunJobs is the job count GET /actions/runs/{id}/jobs reports + // per run id (absent = zero jobs). + workflowRunJobs map[int]int // headCommitAuthor is the login GET /commits/{head} reports as author; // commentBodies collects POST /issues/{n}/comments bodies. headCommitAuthor string @@ -1959,6 +1962,17 @@ func newSelfAuthoredAutoMergeAPI(t *testing.T, prs []selfAuthoredPR, merged *[]i runs = []map[string]any{} } json.NewEncoder(w).Encode(map[string]any{"total_count": len(runs), "workflow_runs": runs}) + case r.Method == http.MethodGet && strings.HasPrefix(r.URL.Path, "/repos/acme/widget/actions/runs/") && strings.HasSuffix(r.URL.Path, "/jobs"): + id := pathNumber(t, r.URL.Path, "/repos/acme/widget/actions/runs/", "/jobs") + count := 0 + for _, pr := range prs { + count += pr.workflowRunJobs[id] + } + jobs := []map[string]any{} + for i := 0; i < count; i++ { + jobs = append(jobs, map[string]any{"id": i + 1, "run_id": id, "status": "queued"}) + } + json.NewEncoder(w).Encode(map[string]any{"total_count": count, "jobs": jobs}) case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/branches/main": json.NewEncoder(w).Encode(map[string]any{"commit": map[string]string{"sha": "basesha"}}) case r.Method == http.MethodGet && r.URL.Path == "/repos/acme/widget/branches/main/protection/required_status_checks": diff --git a/src/pkg/github/automerge/automerge_unauthorized.go b/src/pkg/github/automerge/automerge_unauthorized.go index a872bf1ff8..d72e68fc7d 100644 --- a/src/pkg/github/automerge/automerge_unauthorized.go +++ b/src/pkg/github/automerge/automerge_unauthorized.go @@ -165,6 +165,12 @@ func (c *Engine) clearUnauthorized(ctx context.Context, displayRepo, owner, repo if !had && !labelled { return } + if labelled && hgithub.IntegrationReadOnly(ctx) { + // The read-only lane probe refuses the DELETE; a write-capable lane + // clears the label, so retrying here would only warn every cycle. + c.debug("lane unauthorized: label removal left to a write-capable lane", "repo", displayRepo, "pr", number, "label", unauthorizedLabel, "step", step) + return + } if labelled { if _, err := c.gh.Issues.RemoveLabelForIssue(hgithub.WithRESTCaller(ctx, unauthorizedRESTCaller), owner, repo, number, url.PathEscape(unauthorizedLabel)); err != nil && !isGitHubStatus(err, http.StatusNotFound) { c.warn("lane unauthorized: could not remove label", "repo", displayRepo, "pr", number, "label", unauthorizedLabel, "error", err) diff --git a/src/pkg/github/automerge/automerge_unauthorized_readonly_test.go b/src/pkg/github/automerge/automerge_unauthorized_readonly_test.go new file mode 100644 index 0000000000..765c435926 --- /dev/null +++ b/src/pkg/github/automerge/automerge_unauthorized_readonly_test.go @@ -0,0 +1,58 @@ +package automerge + +import ( + "bytes" + "context" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + gh "github.com/google/go-github/v72/github" + hgithub "github.com/hivecommons/hive/pkg/github" +) + +func TestClearUnauthorizedReadOnlyLaneSkipsLabelRemoval(t *testing.T) { + for _, tc := range []struct { + name string + readOnly bool + }{ + {"read-only lane", true}, + {"write-capable lane", false}, + } { + t.Run(tc.name, func(t *testing.T) { + var deletes atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodDelete && strings.HasSuffix(r.URL.Path, "/issues/7/labels/"+unauthorizedLabel) { + deletes.Add(1) + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`[]`)) + })) + defer srv.Close() + c := newContributorLaneEngine(srv.URL, ContributorLanePolicy{}, nil, Options{}) + var logs bytes.Buffer + c.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelInfo})) + pr := &gh.PullRequest{Labels: []*gh.Label{{Name: gh.Ptr(unauthorizedLabel)}}} + ctx := context.Background() + if tc.readOnly { + ctx = hgithub.WithIntegrationReadOnly(ctx) + } + for range 3 { + c.clearUnauthorized(ctx, "acme/widget", "acme", "widget", pr, 7, "approve-workflow-runs") + } + if tc.readOnly { + if got := deletes.Load(); got != 0 { + t.Fatalf("deletes = %d, want 0", got) + } + } else if deletes.Load() < 1 { + t.Fatal("write-capable lane did not remove the label") + } + if strings.Contains(logs.String(), "level=WARN") { + t.Fatalf("unexpected warning: %s", logs.String()) + } + }) + } +} diff --git a/src/pkg/github/automerge_queue_prow_test.go b/src/pkg/github/automerge_queue_prow_test.go index 701ea76755..5896aef6aa 100644 --- a/src/pkg/github/automerge_queue_prow_test.go +++ b/src/pkg/github/automerge_queue_prow_test.go @@ -1,9 +1,11 @@ package github import ( + "bytes" "context" "encoding/json" "errors" + "log/slog" "net/http" "net/http/httptest" "strings" @@ -186,3 +188,39 @@ func TestBindProwLgtmWarningOutlivesCallerDeadline(t *testing.T) { t.Fatalf("warning comments = %d, want 1 despite the caller's deadline expiring", rec.comments) } } + +// The permission wording is the github_app_statuses_permission_missing +// self-report keyword, so it must name only the 403; any other failure logs +// the neutral line and stays out of the operator-only permission class. +func TestBindProwLgtmWarningNamesPermissionOnlyOn403(t *testing.T) { + for _, tc := range []struct { + status int + wantPerm bool + wantPrefix string + }{ + {status: http.StatusForbidden, wantPerm: true}, + {status: http.StatusBadGateway, wantPerm: false}, + {status: http.StatusUnprocessableEntity, wantPerm: false}, + } { + t.Run(http.StatusText(tc.status), func(t *testing.T) { + rec := &prowQueueRecorder{} + api := prowQueueServer(t, "lgtm", tc.status, rec) + defer api.Close() + var logBuf bytes.Buffer + c := NewClient("token", "acme", []string{"widget"}, slog.New(slog.NewTextHandler(&logBuf, nil)), api.URL) + if err := c.bindProwLgtm(context.Background(), "acme", "widget", 7, "0123456789abcdef", "alice"); err == nil { + t.Fatal("want the status error returned") + } + logs := logBuf.String() + if !strings.Contains(logs, "could not set the prow/lgtm commit status") { + t.Fatalf("status failure not logged: %q", logs) + } + if got := strings.Contains(logs, prowLgtmStatusForbiddenMsg); got != tc.wantPerm { + t.Fatalf("permission wording present = %v, want %v: %q", got, tc.wantPerm, logs) + } + if rec.comments != 1 { + t.Fatalf("warning comments = %d, want 1", rec.comments) + } + }) + } +} diff --git a/src/pkg/github/client.go b/src/pkg/github/client.go index fbd7762c50..7021b813d4 100644 --- a/src/pkg/github/client.go +++ b/src/pkg/github/client.go @@ -3579,8 +3579,22 @@ const ( prowLgtmWarningMarker = "<!-- hive-prow-lgtm-status-warning -->" prowWarningTimeout = 15 * time.Second + + // prowLgtmStatusForbiddenMsg is the WARN logged when GitHub answers the + // prow/lgtm status write with 403. Its text is the keyword behind the + // github_app_statuses_permission_missing self-report class + // (pkg/fleetreport/runtime.go), so it is emitted only for that case. + prowLgtmStatusForbiddenMsg = "could not set the prow/lgtm commit status: the GitHub App lacks the statuses: write permission" ) +// isStatusesPermissionDenied reports whether err is GitHub refusing the +// status write with 403 ("Resource not accessible by integration"), the +// signature of an App installation without Commit statuses read & write. +func isStatusesPermissionDenied(err error) bool { + var ghErr *gh.ErrorResponse + return errors.As(err, &ghErr) && ghErr.Response != nil && ghErr.Response.StatusCode == http.StatusForbidden +} + // bindProwLgtm records the `prow/lgtm` success status on headSHA, the commit // the queue review approved. On failure it returns the error, logs it when a // logger is available, and attempts to surface a warning on the PR. The caller @@ -3611,8 +3625,17 @@ func (c *Client) bindProwLgtm(ctx context.Context, owner, repoName string, numbe return nil } if c.logger != nil { - c.logger.Warn("could not set the prow/lgtm commit status; the GitHub App likely lacks the statuses: write permission", - "repo", owner+"/"+repoName, "number", number, "error", err) + // Only a 403 names the permission gap: that line is the + // github_app_statuses_permission_missing self-report keyword. Any + // other failure (5xx, 422, transport, deadline) stays neutral so a + // transient outage is not filed as an installation-permission fault. + if isStatusesPermissionDenied(err) { + c.logger.Warn(prowLgtmStatusForbiddenMsg, + "repo", owner+"/"+repoName, "number", number, "error", err) + } else { + c.logger.Warn("could not set the prow/lgtm commit status", + "repo", owner+"/"+repoName, "number", number, "error", err) + } } note := "Hive queued this PR but could not set the `prow/lgtm` commit status, so Prow may strip the `lgtm` label and the auto-merge queue may not hold." // A caller that withdrew has nothing left to warn about; a caller whose diff --git a/src/pkg/github/gfi_forge.go b/src/pkg/github/gfi_forge.go index 5f1c22d387..613a0537de 100644 --- a/src/pkg/github/gfi_forge.go +++ b/src/pkg/github/gfi_forge.go @@ -17,6 +17,19 @@ func HardSuppressIssueLabels() []string { return append([]string(nil), hardSuppressIssueLabels...) } +// ArchivedWrite shares the stage sweeps' process-lifetime archive cache. +func (c *Client) ArchivedWrite(repo string, err error) bool { + if c == nil { + return false + } + owner, name := c.splitRepo(repo) + key := strings.ToLower(owner + "/" + name) + if err != nil { + return c.noteArchivedWrite(key, err) + } + return c.stageArchived.has(key) +} + // UpsertIssueMarkerComment implements gfi.Forge: it finds the first comment // on repo#number containing marker, edits it to body when body is non-empty // and differs, and — only when create is set — posts body when none exists. diff --git a/src/pkg/github/gfi_forge_test.go b/src/pkg/github/gfi_forge_test.go index a577f1482c..1a954e73a4 100644 --- a/src/pkg/github/gfi_forge_test.go +++ b/src/pkg/github/gfi_forge_test.go @@ -10,6 +10,7 @@ import ( "sync" "testing" + gh "github.com/google/go-github/v72/github" "github.com/hivecommons/hive/pkg/gfi" ) @@ -29,6 +30,27 @@ func TestHardSuppressIssueLabelsIsCopyWithoutGFI(t *testing.T) { } } +func TestGFIArchivedWriteRecognizesOnlyArchived403(t *testing.T) { + c := NewClientForTest("http://example.com", "acme", []string{"widgets"}, testLogger()) + for _, tc := range []struct { + status int + message string + want bool + }{ + {http.StatusForbidden, "Resource not accessible by integration", false}, + {http.StatusInternalServerError, "Repository was archived", false}, + {http.StatusForbidden, "Repository was archived so is read-only.", true}, + } { + err := &gh.ErrorResponse{Response: &http.Response{StatusCode: tc.status}, Message: tc.message} + if got := c.ArchivedWrite("widgets", err); got != tc.want { + t.Fatalf("status=%d message=%q: got %v", tc.status, tc.message, got) + } + } + if !c.ArchivedWrite("ACME/Widgets", nil) || c.ArchivedWrite("acme/other", nil) { + t.Fatal("archive cache must be case-insensitive and repository-scoped") + } +} + func TestUpsertIssueMarkerComment(t *testing.T) { var mu sync.Mutex comments := []map[string]any{ diff --git a/src/pkg/github/integration_read_only.go b/src/pkg/github/integration_read_only.go index dace3f27ae..ab0fa0f93c 100644 --- a/src/pkg/github/integration_read_only.go +++ b/src/pkg/github/integration_read_only.go @@ -24,6 +24,13 @@ func WithIntegrationReadOnly(ctx context.Context) context.Context { return context.WithValue(ctx, integrationReadOnlyKey{}, true) } +// IntegrationReadOnly reports whether ctx carries WithIntegrationReadOnly, so +// callers can skip writes the transport would refuse anyway. +func IntegrationReadOnly(ctx context.Context) bool { + guarded, _ := ctx.Value(integrationReadOnlyKey{}).(bool) + return guarded +} + func integrationReadOnlyRequest(request *http.Request) error { if guarded, _ := request.Context().Value(integrationReadOnlyKey{}).(bool); !guarded { return nil diff --git a/src/pkg/github/issue_request_watcher.go b/src/pkg/github/issue_request_watcher.go index 91e341bfb7..d3b413c689 100644 --- a/src/pkg/github/issue_request_watcher.go +++ b/src/pkg/github/issue_request_watcher.go @@ -2027,6 +2027,8 @@ func (c *Client) reconcileIssueCreate(ctx context.Context, boundary effects.Boun if expErr != nil { return effects.Result{}, fmt.Errorf("%w (reconciliation lookup failed: %v; expiry failed: %v)", journalErr, lookupErr, expErr) } + // The boundary may already have expired the entry while refusing. + expired = expired || errors.Is(journalErr, effects.ErrUnresolvedExpired) if !expired { return effects.Result{}, fmt.Errorf("%w (reconciliation lookup failed: %v)", journalErr, lookupErr) } diff --git a/src/pkg/github/pr_decision_operator.go b/src/pkg/github/pr_decision_operator.go index 6ca3c432df..8f0e30e6e2 100644 --- a/src/pkg/github/pr_decision_operator.go +++ b/src/pkg/github/pr_decision_operator.go @@ -94,6 +94,13 @@ type PullRequestApprovalEvent struct { // reply stays on the PR, so the next Approve takes the CommandsSkipped // path (hivecommons/hive#17475). ProwRejected bool + // LGTMSkipped is true when only `/approve` was relayed: the PR is + // Prow-managed and was opened by the hive's own account, so a `/lgtm` + // posted from that account is the PR author's and Prow's lgtm plugin + // answers every one with "you cannot LGTM your own PR" + // (hivecommons/hive#17863). No hive lane reads the relayed `/lgtm`; + // the `/approve` comment carries the attribution they use. + LGTMSkipped bool // ReportingError preserves partial progress: the native approval stands, // but comments, labels or stage reporting must be retried/reconciled. ReportingError string @@ -503,19 +510,40 @@ func (c *Client) reportOperatorPullRequestApproval(ctx context.Context, owner, n c.logger.Info("pull request approval commands not posted: Prow rejected the hive's earlier commands on this PR (bot is not an approver); the hive's own approval record stands", "repo", event.Repo, "pr", number, "actor", actor, "head", head) } } else { - for _, command := range PRApprovalCommands { + commands := PRApprovalCommands + if event.ProwManaged && selfAuthored { + commands = prApprovalCommandsWithoutLGTM(commands) + event.LGTMSkipped = true + } + // Each relayed command carries the prApprovalCommentMarkerPrefix + // marker bound to head, which is what prowRejectedHiveCommands and + // the sentinel/workflow-approval readers key on; once posted for a + // head it is not posted again (CommandsSkipped above). + for _, command := range commands { if err := c.CreateIssueComment(ctx, full, number, renderPRApprovalCommandComment(command, actor, head)); err != nil { return fmt.Errorf("posting %s on %s/%s#%d: %w", command, owner, name, number, err) } } if c.logger != nil { - c.logger.Info("pull request approval commands posted by dashboard decision", "repo", event.Repo, "pr", number, "actor", actor, "head", head) + c.logger.Info("pull request approval commands posted by dashboard decision", "repo", event.Repo, "pr", number, "actor", actor, "head", head, "lgtm_skipped_self_authored", event.LGTMSkipped) } event.ProwRejected = c.awaitProwRejection(ctx, owner, name, number, event.ProwManaged) } return ctx.Err() } +// prApprovalCommandsWithoutLGTM returns commands minus `/lgtm`, for a +// Prow-managed PR the posting account itself authored (#17863). +func prApprovalCommandsWithoutLGTM(commands []string) []string { + out := make([]string, 0, len(commands)) + for _, command := range commands { + if command != "/lgtm" { + out = append(out, command) + } + } + return out +} + // ProwManagedLabels reports whether labels carry Prow's plugin markers: // the approve plugin's `approved`, size's `size/*`, the `do-not-merge/*` // family and trigger's `ok-to-test` / `needs-ok-to-test`. diff --git a/src/pkg/github/pr_decision_operator_test.go b/src/pkg/github/pr_decision_operator_test.go index c6a5746ad2..96afbdc68a 100644 --- a/src/pkg/github/pr_decision_operator_test.go +++ b/src/pkg/github/pr_decision_operator_test.go @@ -390,6 +390,42 @@ func TestRecordOperatorPullRequestApprovalSkipsCommandsProwRejected(t *testing.T } } +// On a Prow-managed PR the hive account opened, a `/lgtm` from that account +// is a self-LGTM Prow always rebukes; only `/approve` is relayed +// (hivecommons/hive#17863). A person's PR and a non-Prow repo keep both. +func TestRecordOperatorPullRequestApprovalSkipsSelfLGTMOnProwPR(t *testing.T) { + for _, tc := range []struct { + name string + pr unparkWireIssue + wantLGTM bool + wantSkipped bool + }{ + {"hive-authored prow-managed", parkedPR(liveDecisionPR, "hive[bot]", []string{PRNeedsDecisionStageLabel, "author/hive", "size/M"}), false, true}, + {"hive-authored not prow-managed", hiveParkedPR(), true, false}, + {"person-authored prow-managed", parkedPR(liveDecisionPR, "contributor", []string{PRNeedsDecisionStageLabel, "author/human", "size/M"}), true, false}, + } { + t.Run(tc.name, func(t *testing.T) { + c, rec := newUnparkPRClient(t, []unparkWireIssue{tc.pr}, map[string]string{"maintainer": "write"}) + event, err := c.RecordOperatorPullRequestApproval(context.Background(), "org/repo", liveDecisionPR, "maintainer") + if err != nil { + t.Fatalf("approval: %v", err) + } + if event.LGTMSkipped != tc.wantSkipped { + t.Fatalf("event = %+v, want LGTMSkipped=%v", event, tc.wantSkipped) + } + posted := rec.posted[liveDecisionPR] + var gotLGTM, gotApprove bool + for _, body := range posted { + gotLGTM = gotLGTM || strings.HasPrefix(body, "/lgtm\n") + gotApprove = gotApprove || strings.HasPrefix(body, "/approve\n") + } + if gotLGTM != tc.wantLGTM || !gotApprove { + t.Fatalf("posted %v, want lgtm=%v and /approve", posted, tc.wantLGTM) + } + }) + } +} + // On a Prow repo where rejection has not been observed yet, Prow answers the // relayed commands after they are posted. The approval event reports that // reply instead of leaving the board at "approved" (hivecommons/hive#17475). @@ -418,8 +454,9 @@ func TestRecordOperatorPullRequestApprovalReportsProwRejectionAfterPost(t *testi if err != nil { t.Fatalf("approval: %v", err) } - if event.ProwRejected != tc.want || event.CommandsSkipped != "" || len(rec.posted[liveDecisionPR]) != len(PRApprovalCommands) { - t.Fatalf("event = %+v, want ProwRejected %v with commands posted", event, tc.want) + // Hive-authored and Prow-managed: only `/approve` is relayed (#17863). + if event.ProwRejected != tc.want || event.CommandsSkipped != "" || !event.LGTMSkipped || len(rec.posted[liveDecisionPR]) != len(PRApprovalCommands)-1 { + t.Fatalf("event = %+v, want ProwRejected %v with /approve posted", event, tc.want) } }) } diff --git a/src/pkg/github/pr_signed_reconcile.go b/src/pkg/github/pr_signed_reconcile.go index e2a999337f..b8770a30bd 100644 --- a/src/pkg/github/pr_signed_reconcile.go +++ b/src/pkg/github/pr_signed_reconcile.go @@ -69,6 +69,12 @@ const signedSkipRemedyPerson = "The hive re-signs only its own agents' commits ( "or drop your own commit from the branch so the hive can re-sign the agent's work on a later pass, " + "then push your change again, signed, on top." +// signedSkipRemedyUnsupported applies when the change cannot safely be +// re-authored, rather than a person's commit blocking the rewrite. +const signedSkipRemedyUnsupported = "The hive could not safely represent this change with GitHub's signing API. " + + "Have a contributor re-author the unsigned commits with their own verified signature and push the signed branch, " + + "preserving the file modes and contents. A new head also lets the hive try again." + // signedSkipRemedyAPIRefused tells the PR owner how to unblock a head GitHub // kept refusing to sign: there is no person's commit to squash, only a new // head (or a repository-side fix) re-arms the reconciler. @@ -336,23 +342,25 @@ func (c *Client) reconcileSignedPR(ctx context.Context, owner, repo string, pr * oid, replaced, err := c.signBranch(ctx, owner, repo, base, head) if err != nil { var moved signedHeadMovedError - c.warn("signed-commit reconciler: could not re-author the unsigned tail as a GitHub-signed commit", append(logArgs, "base", base, "reason", err.Error())...) if errors.As(err, &moved) { + c.info("signed-commit reconciler: head moved while signing; will retry", append(logArgs, "base", base, "reason", err.Error())...) return "" // transient: retry next pass } - if !errors.Is(err, errSignedCommitSkip) { - if !signedAPIRefusal(err) { - return "" // outage (5xx, network): retry next pass, uncounted - } - if n := c.signedReconcile.noteFailure(key, headSHA); n < signedReconcileMaxAttempts { - return "" // retry next pass - } - c.warn("signed-commit reconciler: giving up on this head after repeated refusals", append(logArgs, "attempts", signedReconcileMaxAttempts)...) - reason := fmt.Sprintf("GitHub refused to sign this head %d times in a row: %s", signedReconcileMaxAttempts, signedSkipReason(err)) - c.noteSignedSkip(ctx, owner, repo, number, reason, signedSkipRemedyAPIRefused) + if errors.Is(err, errSignedCommitSkip) { + c.info("signed-commit reconciler: cannot safely re-author unsigned tail; leaving branch unchanged", append(logArgs, "base", base, "reason", err.Error())...) + c.noteSignedSkip(ctx, owner, repo, number, err.Error(), signedSkipRemedyUnsupported) return headSHA } - c.noteSignedSkip(ctx, owner, repo, number, err.Error(), signedSkipRemedyPerson) + c.warn("signed-commit reconciler: could not re-author the unsigned tail as a GitHub-signed commit", append(logArgs, "base", base, "reason", err.Error())...) + if !signedAPIRefusal(err) { + return "" // outage (5xx, network): retry next pass, uncounted + } + if n := c.signedReconcile.noteFailure(key, headSHA); n < signedReconcileMaxAttempts { + return "" // retry next pass + } + c.warn("signed-commit reconciler: giving up on this head after repeated refusals", append(logArgs, "attempts", signedReconcileMaxAttempts)...) + reason := fmt.Sprintf("GitHub refused to sign this head %d times in a row: %s", signedReconcileMaxAttempts, signedSkipReason(err)) + c.noteSignedSkip(ctx, owner, repo, number, reason, signedSkipRemedyAPIRefused) return headSHA } c.signedReconcile.clearFailures(key) diff --git a/src/pkg/github/pr_signed_reconcile_test.go b/src/pkg/github/pr_signed_reconcile_test.go index 4245f5bd57..9b65a47495 100644 --- a/src/pkg/github/pr_signed_reconcile_test.go +++ b/src/pkg/github/pr_signed_reconcile_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "log/slog" "net/http" "strings" "testing" @@ -396,6 +397,47 @@ func TestSignedReconcile_UnchangedHeadIsNotRelisted(t *testing.T) { } } +// Unsupported file modes are a deliberate safety refusal, not an API outage. +// Keep the branch intact, explain how to unblock it, and settle the head. +func TestSignedReconcile_UnsupportedModeSettlesWithActionableNote(t *testing.T) { + for _, mode := range []string{"100755", "120000", "160000"} { + t.Run(mode, func(t *testing.T) { + m, c := reconcileFixture(t, true) + m.treeMode["tests/new_test.py"] = mode + m.prCommits = []map[string]any{prCommitJSON(m.headSHA, false, "", "quality@hive.kubestellar.io")} + var logs strings.Builder + c.logger = slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelWarn})) + + c.reconcileSignedCommits(context.Background()) + c.reconcileSignedCommits(context.Background()) + + if m.refPatch != nil || m.graphql != nil || m.callIndex("POST /repos/o/r/git/refs") >= 0 { + t.Fatalf("unsupported mode must leave the branch untouched: %v", m.calls) + } + if m.countCalls(prCommitsCall) != 1 { + t.Errorf("unchanged unsupported head must be settled: %v", m.calls) + } + if len(m.posted) != 1 || !strings.Contains(m.posted[0], mode) || + !strings.Contains(m.posted[0], "their own verified signature") || + strings.Contains(m.posted[0], "drop your own commit") { + t.Fatalf("expected one actionable mode-specific comment, got %q", m.posted) + } + if strings.Contains(logs.String(), "could not re-author the unsigned tail") { + t.Errorf("expected skip must not be reported as an API failure: %s", logs.String()) + } + + newHead := "dddd000000000000000000000000000000000030" + m.headSHA = newHead + m.openPRs = []map[string]any{reconcilePR(77, reconcileBot, newHead)} + m.prCommits = []map[string]any{prCommitJSON(newHead, false, "", "quality@hive.kubestellar.io")} + c.reconcileSignedCommits(context.Background()) + if m.countCalls(prCommitsCall) != 2 || len(m.posted) != 1 { + t.Errorf("new head must be retried without another comment: calls=%v posted=%q", m.calls, m.posted) + } + }) + } +} + // A head that moved while the signed commit was built is transient: no // comment, and the next pass looks again (the open-time skip is retried). func TestSignedReconcile_MovedHeadRetriesNextPass(t *testing.T) { diff --git a/src/pkg/hub/agent_inactivity.go b/src/pkg/hub/agent_inactivity.go index 0cd7fb41e4..3531af4a61 100644 --- a/src/pkg/hub/agent_inactivity.go +++ b/src/pkg/hub/agent_inactivity.go @@ -153,6 +153,13 @@ func classifyInactiveAgent(a AgentSummary, queuedWork int, now time.Time) agentI // turns a healthy fresh digest into "idle with work queued". return agentInactiveNone } + if !legacyAgent(a) && !a.ExpectedActive { + // A new-protocol spoke that says the agent is not expected active in + // the governor's current mode (no cadence, or a paused one — e.g. + // redteam in quiet mode) will never kick it; sitting idle is the + // configured behaviour, not a fault. + return agentInactiveNone + } lastActivity, activityOK := parseAgentTime(a.LastActivityAt) if !activityOK { // The spoke has never observed a pane change for this agent. That is diff --git a/src/pkg/hub/agent_inactivity_test.go b/src/pkg/hub/agent_inactivity_test.go index 39aec10838..a1fbba6c37 100644 --- a/src/pkg/hub/agent_inactivity_test.go +++ b/src/pkg/hub/agent_inactivity_test.go @@ -188,6 +188,27 @@ func TestIdleRuleRespectsKickCadence(t *testing.T) { } } +func TestIdleRuleSkipsOffCadenceAgents(t *testing.T) { + now := time.Now() + a := AgentSummary{ + Name: "redteam", State: "running", Backend: "copilot", + Enabled: true, CanOpenIssue: true, CanOpenPR: true, + StartedAt: settled(now), LastActivityAt: activeAt(now, 6*time.Hour), + } + + // Off-cadence in the governor's current mode (projectbluefin redteam in + // quiet mode): the governor never kicks it, so idleness is by design. + if got := classifyInactiveAgent(a, 19, now); got != agentInactiveNone { + t.Fatalf("off-cadence agent classified %v; want none", got.label()) + } + + // Same agent once the mode gives it a cadence is held to the idle rule. + a.ExpectedActive = true + if got := classifyInactiveAgent(a, 19, now); got != agentInactiveIdleWithWork { + t.Fatalf("expected-active idle agent classified %v; want idle-with-work", got.label()) + } +} + func TestIdleRuleSkipsWriteIncapableAdvisoryAgents(t *testing.T) { now := time.Now() a := AgentSummary{ diff --git a/src/pkg/hub/cluster_host_aliases.go b/src/pkg/hub/cluster_host_aliases.go new file mode 100644 index 0000000000..d3940710a3 --- /dev/null +++ b/src/pkg/hub/cluster_host_aliases.go @@ -0,0 +1,247 @@ +package hub + +import ( + "context" + "encoding/json" + "fmt" + "net" + "sort" + "strings" + "time" +) + +const ( + // hostAliasesReconcileInterval throttles the sweep. Like NET_ADMIN and the + // per-hive env lane this is static remediation: a hive either carries the + // cluster's hostAliases or it does not, and a converged hive stays + // converged until clusters.json changes. + hostAliasesReconcileInterval = 15 * time.Minute + + // hostAliasesKubectlTimeout bounds each per-hive kubectl get/patch so one + // unreachable cluster cannot stall the whole sweep. + hostAliasesKubectlTimeout = 15 * time.Second + + // hostAliasesMaxPatchesPerCycle caps PATCHES per sweep. A hostAliases change + // is a podspec change and rolls the pod; the 2026-11-11 vllm-d rollout + // showed 28 simultaneous rollouts saturating CRI-O on one worker (5.8 GB + // image) and liveness-killing neighbours, so convergence is deliberately + // slow: 3 per 15 min. + hostAliasesMaxPatchesPerCycle = 3 + + // podHostAliasesPath is the JSON-Patch path to the pod-level hostAliases + // list in the hive Deployment. + podHostAliasesPath = "/spec/template/spec/hostAliases" +) + +// ClusterHostAlias is one Pod hostAliases entry the hosted-hive provisioner +// renders into every hive Deployment on the cluster (and the reconcile sweep +// converges onto existing ones). It exists for GitHub Enterprise hosts that +// are reachable only through a TLS REVERSE proxy pinned to a fixed IP — not a +// CONNECT forward proxy, so upstream_proxy cannot express it (#17801). +type ClusterHostAlias struct { + IP string `json:"ip" yaml:"ip"` + Hostnames []string `json:"hostnames" yaml:"hostnames"` +} + +// normalizeClusterHostAliases validates and canonicalises a cluster's +// host_aliases: the IP must parse, every entry needs at least one hostname, +// hostnames are trimmed and lowercased, blanks dropped. Returns nil for an +// empty input so callers can test `len(out) == 0`. +func normalizeClusterHostAliases(in []ClusterHostAlias) ([]ClusterHostAlias, error) { + if len(in) == 0 { + return nil, nil + } + out := make([]ClusterHostAlias, 0, len(in)) + for i, a := range in { + ip := strings.TrimSpace(a.IP) + if net.ParseIP(ip) == nil { + return nil, fmt.Errorf("host_aliases[%d]: ip %q does not parse", i, a.IP) + } + names := make([]string, 0, len(a.Hostnames)) + for _, h := range a.Hostnames { + h = strings.ToLower(strings.TrimSpace(h)) + if h == "" { + continue + } + if strings.ContainsAny(h, " \t/:") { + return nil, fmt.Errorf("host_aliases[%d]: hostname %q is not a bare host", i, h) + } + names = append(names, h) + } + if len(names) == 0 { + return nil, fmt.Errorf("host_aliases[%d]: ip %s has no hostnames", i, ip) + } + out = append(out, ClusterHostAlias{IP: ip, Hostnames: names}) + } + return out, nil +} + +// hostAliasesPatchJSON builds the JSON-Patch that sets the pod-level +// hostAliases to exactly want. "add" on an existing path replaces it, so one +// op serves both the missing and the drifted case. +func hostAliasesPatchJSON(want []ClusterHostAlias) (string, error) { + val, err := json.Marshal(want) + if err != nil { + return "", err + } + patch, err := json.Marshal([]map[string]any{{ + "op": "add", "path": podHostAliasesPath, "value": json.RawMessage(val), + }}) + if err != nil { + return "", err + } + return string(patch), nil +} + +// hostAliasesConverged reports whether the live hostAliases (raw jsonpath +// output, possibly empty) already carry every entry in want. Order and +// hostname case are ignored; EXTRA live entries are tolerated so an operator's +// hand-applied alias for an unrelated host is never clobbered by the sweep. +func hostAliasesConverged(liveRaw []byte, want []ClusterHostAlias) bool { + if len(want) == 0 { + return true + } + var live []ClusterHostAlias + if len(strings.TrimSpace(string(liveRaw))) > 0 { + if err := json.Unmarshal(liveRaw, &live); err != nil { + return false + } + } + liveSet := map[string]bool{} + for _, a := range live { + for _, h := range a.Hostnames { + liveSet[strings.TrimSpace(a.IP)+"|"+strings.ToLower(strings.TrimSpace(h))] = true + } + } + for _, a := range want { + for _, h := range a.Hostnames { + if !liveSet[a.IP+"|"+h] { + return false + } + } + } + return true +} + +// mergeHostAliases returns want merged over live: live entries for IPs not in +// want are kept (so unrelated hand-applied aliases survive), entries for IPs +// in want are replaced by the desired hostnames. Output is sorted by IP so the +// patch is deterministic. +func mergeHostAliases(liveRaw []byte, want []ClusterHostAlias) []ClusterHostAlias { + var live []ClusterHostAlias + if len(strings.TrimSpace(string(liveRaw))) > 0 { + _ = json.Unmarshal(liveRaw, &live) + } + wantIPs := map[string]bool{} + for _, a := range want { + wantIPs[a.IP] = true + } + out := make([]ClusterHostAlias, 0, len(live)+len(want)) + for _, a := range live { + if !wantIPs[strings.TrimSpace(a.IP)] { + out = append(out, a) + } + } + out = append(out, want...) + sort.SliceStable(out, func(i, j int) bool { return out[i].IP < out[j].IP }) + return out +} + +// reconcileHostAliasesIfDue runs the sweep at most once per +// hostAliasesReconcileInterval. Safe to call from the poller loop every tick; +// same guarding mutex as reconcileNetAdminIfDue (poller-loop-only state). +func (s *HubServer) reconcileHostAliasesIfDue() { + s.clusterUnreachableMu.Lock() + due := s.lastHostAliasesReconcile.IsZero() || + time.Since(s.lastHostAliasesReconcile) >= hostAliasesReconcileInterval + if due { + s.lastHostAliasesReconcile = time.Now() + } + s.clusterUnreachableMu.Unlock() + if !due { + return + } + s.reconcileHostAliases() +} + +// reconcileHostAliases sweeps every hosted hive whose cluster declares +// host_aliases and patches the Deployment's pod hostAliases where the desired +// entries are absent — at most hostAliasesMaxPatchesPerCycle per cycle, since +// every patch rolls that hive's pod. Clusters WITHOUT host_aliases are never +// touched, so the lane cannot strip anything an operator applied by hand. +func (s *HubServer) reconcileHostAliases() { + wantByCluster := map[string][]ClusterHostAlias{} + for id, c := range s.clusters { + if len(c.HostAliases) > 0 { + wantByCluster[id] = c.HostAliases + } + } + if len(wantByCluster) == 0 { + return + } + + patched := 0 + deferred := 0 + for _, h := range listSaaSHives() { + if !netAdminSweepEligible(h.Status) { + continue + } + cluster := s.clusterForHive(&h) + if cluster == nil { + continue + } + want, ok := wantByCluster[cluster.ID] + if !ok || s.clusterRecentlyUnreachable(cluster.ID) { + continue + } + ns := hiveHostedNamespacePrefix + h.ID + + ctx, cancel := context.WithTimeout(context.Background(), hostAliasesKubectlTimeout) + getCmd := kubectlForClusterContext(ctx, cluster, "get", "deployment", "hive", + "-n", ns, "-o", "jsonpath={"+strings.ReplaceAll(podHostAliasesPath, "/", ".")+"}") + out, err := getCmd.Output() + cancel() + if err != nil { + s.logger.Debug("host_aliases reconcile: could not read hive deployment", + "hive_id", h.ID, "cluster", cluster.ID, "error", err) + continue + } + s.markClusterReachable(cluster.ID) + + if hostAliasesConverged(out, want) { + continue + } + if patched >= hostAliasesMaxPatchesPerCycle { + deferred++ + continue + } + patch, perr := hostAliasesPatchJSON(mergeHostAliases(out, want)) + if perr != nil { + s.logger.Warn("host_aliases reconcile: could not build patch", + "hive_id", h.ID, "cluster", cluster.ID, "error", perr) + continue + } + pctx, pcancel := context.WithTimeout(context.Background(), hostAliasesKubectlTimeout) + patchCmd := kubectlForClusterContext(pctx, cluster, "patch", "deployment", "hive", + "-n", ns, "--type=json", "-p", patch) + pout, err := patchCmd.CombinedOutput() + pcancel() + if err != nil { + s.markClusterUnreachable(cluster.ID) + s.logger.Warn("host_aliases reconcile: patch failed — will retry next sweep", + "hive_id", h.ID, "cluster", cluster.ID, + "output", strings.TrimSpace(string(pout)), "error", err) + continue + } + s.markClusterReachable(cluster.ID) + patched++ + s.logger.Info("reconciled cluster host_aliases onto hive deployment (rolls the pod once)", + "hive_id", h.ID, "cluster", cluster.ID, "patched_this_cycle", patched) + } + if deferred > 0 { + s.logger.Info("host_aliases reconcile: rate limit reached, remaining hives deferred to next cycle", + "patched", patched, "deferred", deferred, + "max_per_cycle", hostAliasesMaxPatchesPerCycle, + "next_cycle_in", hostAliasesReconcileInterval.String()) + } +} diff --git a/src/pkg/hub/cluster_host_aliases_test.go b/src/pkg/hub/cluster_host_aliases_test.go new file mode 100644 index 0000000000..cbb502f462 --- /dev/null +++ b/src/pkg/hub/cluster_host_aliases_test.go @@ -0,0 +1,150 @@ +package hub + +import ( + "bytes" + "encoding/json" + "log/slog" + "strings" + "testing" + "text/template" +) + +func renderManifestWithHostAliases(t *testing.T, aliases []ClusterHostAlias) string { + t.Helper() + tmpl, err := template.New("manifests").Parse(k8sManifestTemplate) + if err != nil { + t.Fatalf("template does not parse: %v", err) + } + data := map[string]any{ + "ID": "test-hive", "ImageTag": "v2-latest", "ImagePullPolicy": "Always", + "InitContainerUID": "1001", "InitContainerGID": "1000", + "TerminalPort": 7681, "DashboardPort": 8080, + "CPURequest": "500m", "MemRequest": "1Gi", "CPULimit": "2", "MemLimit": "4Gi", + "Namespace": "hive-test", "StorageSize": "10Gi", "StorageClass": "standard", + "HeartbeatKey": "hb", "SessionKey": "sk", "SSOPublicKey": "pk", "TerminalKey": "tk", + "AuthorizedUsers": "alice:owner", "Host": "test-hive.example", "Owner": "alice", + "HubPublicURL": "https://hub.example", "ProjectName": "test", "PrimaryRepo": "org/repo", + "AppKeys": []provisionAppKey{}, + "HostAliases": aliases, + } + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + t.Fatalf("template does not execute: %v", err) + } + return buf.String() +} + +func TestClusterHostAliasesRenderedIntoPodSpec(t *testing.T) { + m := renderManifestWithHostAliases(t, []ClusterHostAlias{ + {IP: "129.34.104.21", Hostnames: []string{"github.ibm.com", "api.github.ibm.com"}}, + }) + want := " spec:\n hostAliases:\n - ip: \"129.34.104.21\"\n hostnames:\n - \"github.ibm.com\"\n - \"api.github.ibm.com\"\n" + // The template carries a comment line between spec: and hostAliases:. + stripped := strings.Join(func() []string { + var out []string + for _, l := range strings.Split(m, "\n") { + if strings.HasPrefix(strings.TrimSpace(l), "#") { + continue + } + out = append(out, l) + } + return out + }(), "\n") + if !strings.Contains(stripped, want) { + t.Fatalf("rendered manifest missing pod hostAliases block:\n%s\n--- got ---\n%s", want, m) + } +} + +func TestClusterHostAliasesOmittedWhenUnset(t *testing.T) { + for name, aliases := range map[string][]ClusterHostAlias{"nil": nil, "empty": {}} { + if m := renderManifestWithHostAliases(t, aliases); strings.Contains(m, "hostAliases") { + t.Errorf("%s: hostAliases must not be rendered when no aliases configured", name) + } + } +} + +func TestNormalizeClusterHostAliases(t *testing.T) { + got, err := normalizeClusterHostAliases([]ClusterHostAlias{ + {IP: " 129.34.104.21 ", Hostnames: []string{" GitHub.IBM.com ", "", "api.github.ibm.com"}}, + }) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if len(got) != 1 || got[0].IP != "129.34.104.21" || + strings.Join(got[0].Hostnames, ",") != "github.ibm.com,api.github.ibm.com" { + t.Fatalf("unexpected normalization: %+v", got) + } + if out, err := normalizeClusterHostAliases(nil); err != nil || out != nil { + t.Fatalf("nil input should yield nil,nil; got %v,%v", out, err) + } + for name, in := range map[string][]ClusterHostAlias{ + "bad ip": {{IP: "not-an-ip", Hostnames: []string{"a.example"}}}, + "no hostnames": {{IP: "10.0.0.1", Hostnames: []string{" ", ""}}}, + "url not host": {{IP: "10.0.0.1", Hostnames: []string{"https://a.example"}}}, + } { + if _, err := normalizeClusterHostAliases(in); err == nil { + t.Errorf("%s: expected error", name) + } + } +} + +func TestValidateClusterEntriesKeepsClusterOnBadHostAliases(t *testing.T) { + logger := slog.New(slog.NewTextHandler(&bytes.Buffer{}, nil)) + got := validateClusterEntries([]ClusterConfig{ + {ID: "bad", InCluster: true, Domain: "x.example", + HostAliases: []ClusterHostAlias{{IP: "nope", Hostnames: []string{"a"}}}}, + {ID: "good", InCluster: true, Domain: "x.example", + HostAliases: []ClusterHostAlias{{IP: "10.1.2.3", Hostnames: []string{"A.Example"}}}}, + }, logger) + bad, ok := got["bad"] + if !ok { + t.Fatal("cluster with invalid host_aliases must still be admitted") + } + if bad.HostAliases != nil { + t.Fatalf("invalid host_aliases must be dropped, got %+v", bad.HostAliases) + } + if g := got["good"].HostAliases; len(g) != 1 || g[0].Hostnames[0] != "a.example" { + t.Fatalf("valid host_aliases must be normalized in registry, got %+v", g) + } +} + +func TestHostAliasesConvergedAndMerge(t *testing.T) { + want := []ClusterHostAlias{{IP: "129.34.104.21", Hostnames: []string{"github.ibm.com", "api.github.ibm.com"}}} + + if hostAliasesConverged(nil, want) { + t.Error("empty live must not be converged") + } + live := []byte(`[{"ip":"129.34.104.21","hostnames":["API.github.ibm.com","github.ibm.com"]},{"ip":"10.0.0.9","hostnames":["other.example"]}]`) + if !hostAliasesConverged(live, want) { + t.Error("live superset (any order/case) must be converged") + } + partial := []byte(`[{"ip":"129.34.104.21","hostnames":["github.ibm.com"]}]`) + if hostAliasesConverged(partial, want) { + t.Error("missing hostname must not be converged") + } + if !hostAliasesConverged([]byte(`[]`), nil) { + t.Error("no desired aliases is always converged") + } + + merged := mergeHostAliases([]byte(`[{"ip":"10.0.0.9","hostnames":["other.example"]},{"ip":"129.34.104.21","hostnames":["stale.example"]}]`), want) + if len(merged) != 2 || merged[0].IP != "10.0.0.9" || merged[1].IP != "129.34.104.21" || + strings.Join(merged[1].Hostnames, ",") != "github.ibm.com,api.github.ibm.com" { + t.Fatalf("merge must keep unrelated IPs and replace desired IPs: %+v", merged) + } + + patch, err := hostAliasesPatchJSON(merged) + if err != nil { + t.Fatal(err) + } + var ops []struct { + Op string `json:"op"` + Path string `json:"path"` + Value []ClusterHostAlias `json:"value"` + } + if err := json.Unmarshal([]byte(patch), &ops); err != nil { + t.Fatalf("patch is not valid JSON-Patch: %v\n%s", err, patch) + } + if len(ops) != 1 || ops[0].Op != "add" || ops[0].Path != podHostAliasesPath || len(ops[0].Value) != 2 { + t.Fatalf("unexpected patch: %s", patch) + } +} diff --git a/src/pkg/hub/clusters_registry.go b/src/pkg/hub/clusters_registry.go index e07495bc6e..d8fb409077 100644 --- a/src/pkg/hub/clusters_registry.go +++ b/src/pkg/hub/clusters_registry.go @@ -207,6 +207,16 @@ func validateClusterEntries(configs []ClusterConfig, logger *slog.Logger) map[st logger.Warn("skipping cluster with no domain", "cluster", c.ID) continue } + // host_aliases is validated but NOT an admission rule: a typo there + // must not drop the cluster (and strand its hives); the aliases are + // dropped instead and the operator told why. + aliases, err := normalizeClusterHostAliases(c.HostAliases) + if err != nil { + logger.Warn("ignoring invalid host_aliases for cluster — hives will NOT get them", + "cluster", c.ID, "error", err) + aliases = nil + } + c.HostAliases = aliases clusters[c.ID] = c } return clusters diff --git a/src/pkg/hub/marketplace.go b/src/pkg/hub/marketplace.go index 19750e96aa..c886fcac26 100644 --- a/src/pkg/hub/marketplace.go +++ b/src/pkg/hub/marketplace.go @@ -180,6 +180,13 @@ func (s *HubServer) handleMarketplacePublish(w http.ResponseWriter, r *http.Requ marketplaceJSONError(w, http.StatusBadRequest, "unknown tier "+b.Tier) return } + // The model's own provenance is hash-covered, so it cannot be normalised + // like the envelope tier; a vouched tier claimed there needs the same + // admin the envelope does, otherwise importers would store it as vouched. + if p := string(b.Model.Spec.Meta.Provenance); (p == bundle.TierCurated || p == bundle.TierBuiltIn) && !who.admin { + marketplaceJSONError(w, http.StatusForbidden, fmt.Sprintf("model provenance %q requires a hub admin", p)) + return + } if err := bundle.Verify(b); err != nil { marketplaceJSONError(w, http.StatusUnprocessableEntity, "bundle verification failed: "+err.Error()) return @@ -215,7 +222,11 @@ func (s *HubServer) handleMarketplacePublish(w http.ResponseWriter, r *http.Requ // catalogue" button works through the spoke. Spokes are rate-limited (one // full run per marketplaceFullRefreshInterval across all spokes, // marketplaceRepoRefreshBurst single-repo runs per minute) → 429 with -// Retry-After; admins are not. +// Retry-After; admins are not. A spoke's ?repo= must also name a repository +// the hub already tracks (the corpus or HIVE_HUB_MARKETPLACE_EXTRA_REPOS), +// otherwise 403: the ingest runs with the hub's GitHub identity and +// publishes under the hub's name, so only an admin may widen what it +// fetches. func (s *HubServer) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Request) { if !s.marketplaceGate(w) { return @@ -234,6 +245,10 @@ func (s *HubServer) handleMarketplaceRefresh(w http.ResponseWriter, r *http.Requ return } if !who.admin { + if !marketplaceTracksRepo(repo) { + marketplaceJSONError(w, http.StatusForbidden, "repo is not in the marketplace corpus; only a hub admin may refresh other repositories") + return + } if ok, wait := rf.allowRepoRefresh(now); !ok { marketplaceTooMany(w, wait, "single-repository refresh") return diff --git a/src/pkg/hub/marketplace_refresh.go b/src/pkg/hub/marketplace_refresh.go index baf5d208a7..f55dd82fc0 100644 --- a/src/pkg/hub/marketplace_refresh.go +++ b/src/pkg/hub/marketplace_refresh.go @@ -192,6 +192,18 @@ func marketplaceRepos() []string { return out } +// marketplaceTracksRepo reports whether repo (already lower-cased and +// trimmed) is one the hub ingests on its own: a corpus entry or an +// HIVE_HUB_MARKETPLACE_EXTRA_REPOS entry. +func marketplaceTracksRepo(repo string) bool { + for _, r := range marketplaceRepos() { + if r == repo { + return true + } + } + return false +} + // marketplaceDisabled reports HIVE_HUB_MARKETPLACE_DISABLED. func marketplaceDisabled() bool { v := strings.ToLower(strings.TrimSpace(os.Getenv(hubMarketplaceDisableEnv))) diff --git a/src/pkg/hub/marketplace_test.go b/src/pkg/hub/marketplace_test.go index c3b8e424de..b364fea6dd 100644 --- a/src/pkg/hub/marketplace_test.go +++ b/src/pkg/hub/marketplace_test.go @@ -320,6 +320,18 @@ func TestMarketplacePublishAuth(t *testing.T) { if rec := mktDo(s, mktSpokeReq(mktBody(t, mktBundle(t, bundle.TierCurated)))); rec.Code != 403 { t.Fatalf("spoke curated: %d %s", rec.Code, rec.Body.String()) } + // Spoke asserting curated inside the hash-covered model (envelope tier + // legitimate) → 403: the hub cannot normalise it, so it must refuse. + launder := mktModel(t) + launder.Spec.Meta.Provenance = workflowmodel.ProvenanceCurated + lb, err := bundle.Export(launder, "spoke", time.Now()) + if err != nil { + t.Fatal(err) + } + lb.Tier = bundle.TierLocalPublished + if rec := mktDo(s, mktSpokeReq(mktBody(t, lb))); rec.Code != 403 || !strings.Contains(rec.Body.String(), "model provenance") { + t.Fatalf("spoke model-curated: %d %s", rec.Code, rec.Body.String()) + } // Spoke OK: `local` is normalised to local-published. rec := mktDo(s, mktSpokeReq(body)) if rec.Code != 201 { @@ -784,7 +796,7 @@ func TestMarketplaceRefreshAuthMatrixAndVerboseStatus(t *testing.T) { s := marketplaceTestHub(t, fetch) clock := s.refresher().now() s.refresher().now = func() time.Time { return clock } - t.Setenv(hubMarketplaceExtraReposEnv, "broken/repo") + t.Setenv(hubMarketplaceExtraReposEnv, "broken/repo,ok/repo") spoke := func(target string) *http.Request { req := httptest.NewRequest(http.MethodPost, target, nil) @@ -819,8 +831,21 @@ func TestMarketplaceRefreshAuthMatrixAndVerboseStatus(t *testing.T) { t.Fatalf("bad bearer: %d", rec.Code) } + // Spoke single repo outside the corpus: refused before any fetch or + // rate-limit accounting; the same request as admin is allowed. + rec := mktDo(s, spoke("/api/marketplace/refresh?repo=outside/corpus")) + if rec.Code != 403 || !strings.Contains(rec.Body.String(), "not in the marketplace corpus") { + t.Fatalf("spoke outside corpus: %d %s", rec.Code, rec.Body.String()) + } + if rec = mktDo(s, admin(http.MethodPost, "/api/marketplace/refresh?repo=outside/corpus")); rec.Code != 200 { + t.Fatalf("admin outside corpus: %d %s", rec.Code, rec.Body.String()) + } + if _, ok := s.marketplace().refreshRecord("outside/corpus"); !ok { + t.Fatal("admin refresh of an outside-corpus repo should have run") + } + // Spoke single repo: ok, then burst-limited. - rec := mktDo(s, spoke("/api/marketplace/refresh?repo=broken/repo")) + rec = mktDo(s, spoke("/api/marketplace/refresh?repo=broken/repo")) if rec.Code != 200 || !strings.Contains(rec.Body.String(), "404 Not Found") { t.Fatalf("spoke single: %d %s", rec.Code, rec.Body.String()) } @@ -845,18 +870,29 @@ func TestMarketplaceRefreshAuthMatrixAndVerboseStatus(t *testing.T) { // Spoke full run: 202 once per hour across callers; admin unaffected. rf := s.refresher() - waitIdle := func() { + waitRefresh := func(previous *marketplaceRefreshResult) { + t.Helper() testutil.Eventually(t, 5*time.Second, func() bool { rf.mu.Lock() defer rf.mu.Unlock() - return !rf.running + return rf.last != previous && !rf.running }, "background refresh did not finish") } - rec = mktDo(s, spoke("/api/marketplace/refresh")) - if rec.Code != 202 { - t.Fatalf("spoke full: %d %s", rec.Code, rec.Body.String()) + refreshAndWait := func(req *http.Request, description string) { + t.Helper() + rf.mu.Lock() + previous := rf.last + rf.mu.Unlock() + rec := mktDo(s, req) + if rec.Code != http.StatusAccepted { + t.Fatalf("%s: %d %s", description, rec.Code, rec.Body.String()) + } + // Even a later fatal assertion must not restore fixture globals while + // this accepted refresh is still starting or running. + t.Cleanup(func() { waitRefresh(previous) }) + waitRefresh(previous) } - waitIdle() + refreshAndWait(spoke("/api/marketplace/refresh"), "spoke full") rf.mu.Lock() trigger := rf.last.Trigger rf.mu.Unlock() @@ -867,15 +903,9 @@ func TestMarketplaceRefreshAuthMatrixAndVerboseStatus(t *testing.T) { if rec.Code != 429 || !strings.Contains(rec.Body.String(), "try again in") { t.Fatalf("spoke full again: %d %s", rec.Code, rec.Body.String()) } - if rec = mktDo(s, admin(http.MethodPost, "/api/marketplace/refresh")); rec.Code != 202 { - t.Fatalf("admin full while spoke limited: %d %s", rec.Code, rec.Body.String()) - } - waitIdle() + refreshAndWait(admin(http.MethodPost, "/api/marketplace/refresh"), "admin full while spoke limited") clock = clock.Add(marketplaceFullRefreshInterval) - if rec = mktDo(s, spoke("/api/marketplace/refresh")); rec.Code != 202 { - t.Fatalf("spoke full after interval: %d", rec.Code) - } - waitIdle() + refreshAndWait(spoke("/api/marketplace/refresh"), "spoke full after interval") // Verbose status: anon 401, spoke and admin get the failing list. if rec = mktDo(s, httptest.NewRequest(http.MethodGet, "/api/marketplace/status?verbose=1", nil)); rec.Code != 401 { diff --git a/src/pkg/hub/release_notes.go b/src/pkg/hub/release_notes.go index 82bc94d3e2..8e621cd090 100644 --- a/src/pkg/hub/release_notes.go +++ b/src/pkg/hub/release_notes.go @@ -13,6 +13,7 @@ import ( "sync" "time" + hgithub "github.com/hivecommons/hive/pkg/github" "github.com/hivecommons/hive/pkg/releasenotes" ) @@ -30,7 +31,9 @@ const ( var ( hubReleaseNotesRawBaseURL = "https://raw.githubusercontent.com" - hubReleaseNotesHTTP = &http.Client{Timeout: 10 * time.Second} + // Contents reads go through pkg/github's transport so repeat fetches of an + // unchanged ref are ETag 304s and the calls show up in /api/gh-rate-limits. + hubReleaseNotesHTTP = &http.Client{Transport: hgithub.NewHTTPTransport(nil), Timeout: 10 * time.Second} hubReleaseNotesSHAPattern = regexp.MustCompile(`^[0-9a-fA-F]{7,40}$`) ) diff --git a/src/pkg/hub/saas_auth.go b/src/pkg/hub/saas_auth.go index d493b6fee6..2daf88d43a 100644 --- a/src/pkg/hub/saas_auth.go +++ b/src/pkg/hub/saas_auth.go @@ -8,11 +8,14 @@ import ( "net/url" "os" "path/filepath" + "strconv" "strings" "sync" "time" "golang.org/x/net/publicsuffix" + + hgithub "github.com/hivecommons/hive/pkg/github" ) func (s *HubServer) requireAuth(next http.HandlerFunc) http.HandlerFunc { @@ -691,13 +694,83 @@ var ( ghTokenCache = map[string]ghTokenCacheEntry{} ) -const ghTokenCacheTTL = 5 * time.Minute +const ( + ghTokenCacheMaxEntries = 10_000 + ghTokenCacheTTL = 5 * time.Minute + // ghTokenNegativeTTL bounds how long a rejected bearer token (401/403, + // including a rate-limited one) is remembered without re-probing + // GET /user. Without it every request carrying a dead or exhausted token + // re-hit GitHub, charging the token owner's own core quota while it was + // already at zero and stretching the outage. + ghTokenNegativeTTL = 60 * time.Second + // ghTokenNegativeMaxTTL caps how far a 403's X-RateLimit-Reset may push + // the negative entry so a skewed clock cannot lock a user out for long. + ghTokenNegativeMaxTTL = 15 * time.Minute + ghTokenValidateTimeout = 10 * time.Second + ghTokenValidateCaller = "agent:hub-bearer-validate" +) type ghTokenCacheEntry struct { username string expiresAt time.Time } +// cacheGHToken bounds both positive and negative entries. At capacity, expired +// entries are reclaimed; if all entries are live, new tokens are not cached. +func cacheGHToken(token string, entry ghTokenCacheEntry) { + ghTokenCacheMu.Lock() + defer ghTokenCacheMu.Unlock() + + if len(ghTokenCache) >= ghTokenCacheMaxEntries { + now := time.Now() + for key, cached := range ghTokenCache { + if !now.Before(cached.expiresAt) { + delete(ghTokenCache, key) + } + } + if len(ghTokenCache) >= ghTokenCacheMaxEntries { + if _, exists := ghTokenCache[token]; !exists { + return + } + } + } + ghTokenCache[token] = entry +} + +// ghTokenValidateHTTPClient probes GET /user for bearer validation. It is a +// plain client on purpose: the probe carries the CALLER's token, and routing +// it through pkg/github's shared transport chain would let one exhausted user +// bucket push the hub's own App traffic into rate-limit caution. The probe is +// still recorded in REST accounting (see validateGitHubToken). +func ghTokenValidateHTTPClient() *http.Client { + return &http.Client{Timeout: ghTokenValidateTimeout} +} + +// ghTokenNegativeExpiry picks the expiry of a negative cache entry for a +// rejected probe: the default short TTL, extended up to ghTokenNegativeMaxTTL +// when GitHub says the token is rate-limited until X-RateLimit-Reset. +func ghTokenNegativeExpiry(status int, hdr http.Header, now time.Time) time.Time { + exp := now.Add(ghTokenNegativeTTL) + if status != http.StatusForbidden && status != http.StatusTooManyRequests { + return exp + } + if hdr.Get("X-RateLimit-Remaining") != "0" { + return exp + } + reset, err := strconv.ParseInt(strings.TrimSpace(hdr.Get("X-RateLimit-Reset")), 10, 64) + if err != nil { + return exp + } + until := time.Unix(reset, 0) + if until.After(now.Add(ghTokenNegativeMaxTTL)) { + until = now.Add(ghTokenNegativeMaxTTL) + } + if until.After(exp) { + exp = until + } + return exp +} + func (s *HubServer) validateGitHubToken(token string) string { if token == "" { return "" @@ -710,19 +783,29 @@ func (s *HubServer) validateGitHubToken(token string) string { } ghTokenCacheMu.RUnlock() - client := &http.Client{Timeout: 10 * time.Second} + client := ghTokenValidateHTTPClient() // Hub always validates tokens against github.com (the hub is a SaaS service). - req, err := http.NewRequest("GET", defaultGHUserURL, nil) + req, err := http.NewRequest(http.MethodGet, defaultGHUserURL, nil) if err != nil { return "" } req.Header.Set("Authorization", "Bearer "+token) req.Header.Set("Accept", "application/vnd.github+json") resp, err := client.Do(req) - if err != nil || resp.StatusCode != 200 { + if err != nil { return "" } defer func() { _ = resp.Body.Close() }() + // Attributed under the pass-through ("agent:") caller prefix: visible in + // /api/gh-rate-limits, but the caller's rate-limit headers never overwrite + // the hub's own core-bucket snapshot. + hgithub.RecordRESTRequest(ghTokenValidateCaller, req.Method, req.URL.Path, resp.StatusCode, resp.Header) + if resp.StatusCode != http.StatusOK { + if resp.StatusCode == http.StatusUnauthorized || resp.StatusCode == http.StatusForbidden || resp.StatusCode == http.StatusTooManyRequests { + cacheGHToken(token, ghTokenCacheEntry{expiresAt: ghTokenNegativeExpiry(resp.StatusCode, resp.Header, time.Now())}) + } + return "" + } var user struct { Login string `json:"login"` } @@ -730,9 +813,7 @@ func (s *HubServer) validateGitHubToken(token string) string { return "" } - ghTokenCacheMu.Lock() - ghTokenCache[token] = ghTokenCacheEntry{username: user.Login, expiresAt: time.Now().Add(ghTokenCacheTTL)} - ghTokenCacheMu.Unlock() + cacheGHToken(token, ghTokenCacheEntry{username: user.Login, expiresAt: time.Now().Add(ghTokenCacheTTL)}) return user.Login } diff --git a/src/pkg/hub/saas_auth_bearer_cache_test.go b/src/pkg/hub/saas_auth_bearer_cache_test.go new file mode 100644 index 0000000000..942b8ba717 --- /dev/null +++ b/src/pkg/hub/saas_auth_bearer_cache_test.go @@ -0,0 +1,227 @@ +package hub + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "sync/atomic" + "testing" + "time" + + hgithub "github.com/hivecommons/hive/pkg/github" +) + +func withBearerProbeServer(t *testing.T, handler http.HandlerFunc) *int32 { + t.Helper() + var hits int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&hits, 1) + handler(w, r) + })) + oldURL := defaultGHUserURL + defaultGHUserURL = srv.URL + "/user" + t.Cleanup(func() { + defaultGHUserURL = oldURL + srv.Close() + }) + return &hits +} + +func clearGHTokenCache(t *testing.T, tokens ...string) { + t.Helper() + clear := func() { + ghTokenCacheMu.Lock() + for _, tok := range tokens { + delete(ghTokenCache, tok) + } + ghTokenCacheMu.Unlock() + } + clear() + t.Cleanup(clear) +} + +// A rejected bearer token must not re-probe GET /user on every request while +// the negative entry is fresh: that re-probe charged the token owner's own +// core quota while it was already exhausted (the 401 storm in the PAT outage). +func TestValidateGitHubToken_RejectedTokenIsNegativelyCached(t *testing.T) { + hits := withBearerProbeServer(t, func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusUnauthorized) + }) + const token = "dead-token-negative-cache" + clearGHTokenCache(t, token) + + srv := newHubServerForTest(t) + for i := 0; i < 3; i++ { + if got := srv.validateGitHubToken(token); got != "" { + t.Fatalf("call %d: rejected token resolved to %q", i, got) + } + } + if n := atomic.LoadInt32(hits); n != 1 { + t.Fatalf("GET /user probed %d times for a rejected token; want 1", n) + } + ghTokenCacheMu.RLock() + entry, ok := ghTokenCache[token] + ghTokenCacheMu.RUnlock() + if !ok || entry.username != "" || !entry.expiresAt.After(time.Now()) { + t.Fatalf("negative entry = %+v, ok=%v; want an unexpired entry with empty username", entry, ok) + } +} + +func TestValidateGitHubToken_CacheCapacityAndExpiry(t *testing.T) { + ghTokenCacheMu.Lock() + original := ghTokenCache + ghTokenCache = make(map[string]ghTokenCacheEntry) + ghTokenCacheMu.Unlock() + t.Cleanup(func() { + ghTokenCacheMu.Lock() + ghTokenCache = original + ghTokenCacheMu.Unlock() + }) + + hits := withBearerProbeServer(t, func(w http.ResponseWriter, r *http.Request) { + if r.Header.Get("Authorization") == "Bearer valid-overflow" { + _ = json.NewEncoder(w).Encode(map[string]string{"login": "valid-user"}) + return + } + w.WriteHeader(http.StatusUnauthorized) + }) + srv := newHubServerForTest(t) + for i := range ghTokenCacheMaxEntries + 1 { + token := "rejected-capacity-" + strconv.Itoa(i) + if got := srv.validateGitHubToken(token); got != "" { + t.Fatalf("rejected token resolved to %q", got) + } + } + ghTokenCacheMu.RLock() + size := len(ghTokenCache) + ghTokenCacheMu.RUnlock() + if size != ghTokenCacheMaxEntries { + t.Fatalf("cache has %d entries, want %d", size, ghTokenCacheMaxEntries) + } + + // Both rejected and accepted tokens still probe when the cache is full. + for range 2 { + if got := srv.validateGitHubToken("rejected-capacity-" + strconv.Itoa(ghTokenCacheMaxEntries)); got != "" { + t.Fatalf("overflow rejected token resolved to %q", got) + } + if got := srv.validateGitHubToken("valid-overflow"); got != "valid-user" { + t.Fatalf("overflow accepted token resolved to %q", got) + } + } + if got, want := atomic.LoadInt32(hits), int32(ghTokenCacheMaxEntries+5); got != want { + t.Fatalf("GET /user probed %d times, want %d", got, want) + } + ghTokenCacheMu.Lock() + entry := ghTokenCache["rejected-capacity-0"] + entry.expiresAt = time.Now().Add(-time.Second) + ghTokenCache["rejected-capacity-0"] = entry + ghTokenCacheMu.Unlock() + + // A full cache sweeps expired entries and makes room for a new positive entry. + for range 2 { + if got := srv.validateGitHubToken("valid-overflow"); got != "valid-user" { + t.Fatalf("accepted token after expiry resolved to %q", got) + } + } + if got, want := atomic.LoadInt32(hits), int32(ghTokenCacheMaxEntries+6); got != want { + t.Fatalf("GET /user probed %d times after expiry, want %d", got, want) + } + ghTokenCacheMu.RLock() + _, expiredPresent := ghTokenCache["rejected-capacity-0"] + size = len(ghTokenCache) + ghTokenCacheMu.RUnlock() + if expiredPresent || size != ghTokenCacheMaxEntries { + t.Fatalf("after sweep: expired present = %t, cache size = %d", expiredPresent, size) + } +} + +// An expired negative entry probes again, and a token that has since become +// valid resolves — the negative cache is a short hold, not a ban. +func TestValidateGitHubToken_ExpiredNegativeEntryReprobes(t *testing.T) { + hits := withBearerProbeServer(t, func(w http.ResponseWriter, r *http.Request) { + _ = json.NewEncoder(w).Encode(map[string]string{"login": "revived"}) + }) + const token = "revived-token" + clearGHTokenCache(t, token) + ghTokenCacheMu.Lock() + ghTokenCache[token] = ghTokenCacheEntry{expiresAt: time.Now().Add(-time.Second)} + ghTokenCacheMu.Unlock() + + srv := newHubServerForTest(t) + if got := srv.validateGitHubToken(token); got != "revived" { + t.Fatalf("validateGitHubToken = %q, want revived", got) + } + if n := atomic.LoadInt32(hits); n != 1 { + t.Fatalf("GET /user probed %d times; want 1", n) + } +} + +func TestGhTokenNegativeExpiry(t *testing.T) { + now := time.Unix(1_700_000_000, 0) + hdr := func(remaining string, reset time.Time) http.Header { + h := http.Header{} + h.Set("X-RateLimit-Remaining", remaining) + h.Set("X-RateLimit-Reset", strconv.FormatInt(reset.Unix(), 10)) + return h + } + cases := []struct { + name string + status int + hdr http.Header + want time.Time + }{ + {"401 uses the short hold", http.StatusUnauthorized, http.Header{}, now.Add(ghTokenNegativeTTL)}, + {"403 with quota left is not a rate limit", http.StatusForbidden, hdr("12", now.Add(time.Hour)), now.Add(ghTokenNegativeTTL)}, + {"403 exhausted holds until reset", http.StatusForbidden, hdr("0", now.Add(5*time.Minute)), now.Add(5 * time.Minute)}, + {"429 exhausted holds until reset", http.StatusTooManyRequests, hdr("0", now.Add(3*time.Minute)), now.Add(3 * time.Minute)}, + {"reset far away is capped", http.StatusForbidden, hdr("0", now.Add(2*time.Hour)), now.Add(ghTokenNegativeMaxTTL)}, + {"reset already passed keeps the short hold", http.StatusForbidden, hdr("0", now.Add(-time.Minute)), now.Add(ghTokenNegativeTTL)}, + {"garbage reset keeps the short hold", http.StatusForbidden, func() http.Header { + h := http.Header{} + h.Set("X-RateLimit-Remaining", "0") + h.Set("X-RateLimit-Reset", "soon") + return h + }(), now.Add(ghTokenNegativeTTL)}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := ghTokenNegativeExpiry(tc.status, tc.hdr, now); !got.Equal(tc.want) { + t.Fatalf("ghTokenNegativeExpiry = %v, want %v", got, tc.want) + } + }) + } +} + +// The probe is attributed in /api/gh-rate-limits under the pass-through +// ("agent:") caller so an exhausted user token is visible on the hub admin +// page without being mistaken for the hub's own App credential. +func TestValidateGitHubToken_ProbeIsAccountedAsPassThroughCaller(t *testing.T) { + withBearerProbeServer(t, func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("X-RateLimit-Limit", "5000") + w.Header().Set("X-RateLimit-Remaining", "0") + w.Header().Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(time.Minute).Unix(), 10)) + w.WriteHeader(http.StatusForbidden) + }) + const token = "accounted-probe-token" + clearGHTokenCache(t, token) + + srv := newHubServerForTest(t) + if got := srv.validateGitHubToken(token); got != "" { + t.Fatalf("validateGitHubToken = %q, want empty", got) + } + var found *hgithub.RESTConsumer + for _, c := range hgithub.RESTTopConsumers(100) { + if c.Caller == ghTokenValidateCaller && c.Endpoint == "/user" { + c := c + found = &c + break + } + } + if found == nil { + t.Fatalf("caller %q not found in REST accounting: %+v", ghTokenValidateCaller, hgithub.RESTTopConsumers(100)) + } + if found.RateLimited == 0 { + t.Fatalf("probe not recorded as rate-limited: %+v", *found) + } +} diff --git a/src/pkg/hub/saas_provision.go b/src/pkg/hub/saas_provision.go index abe671797c..75939a069a 100644 --- a/src/pkg/hub/saas_provision.go +++ b/src/pkg/hub/saas_provision.go @@ -245,6 +245,13 @@ type ClusterConfig struct { // reachable only via the proxy). Injected into new pool slots as // HIVE_UPSTREAM_PROXY_URL / HIVE_UPSTREAM_PROXY_HOSTS / HIVE_NO_PROXY. UpstreamProxy *ClusterUpstreamProxy `json:"upstream_proxy,omitempty" yaml:"upstream_proxy,omitempty"` + // HostAliases are Pod hostAliases entries rendered into every hive + // Deployment on THIS cluster, for GitHub Enterprise hosts reachable only + // through a TLS reverse proxy pinned to a fixed IP (no CONNECT proxy, so + // upstream_proxy cannot express it). Validated at load by + // normalizeClusterHostAliases; existing hives are converged by the + // host_aliases reconcile sweep (cluster_host_aliases.go, #17801). + HostAliases []ClusterHostAlias `json:"host_aliases,omitempty" yaml:"host_aliases,omitempty"` // GitHubAppSlug is the GitHub App's URL slug on THIS cluster's GitHub // host. A GitHub Enterprise instance hosts its own App registration, // which is rarely named "kubestellar-hive" — without this the spoke's @@ -2680,6 +2687,7 @@ func provisionHive(h *SaaSHive, req *CreateHiveRequest, cluster *ClusterConfig, "UpstreamProxyURL": cluster.UpstreamProxy.url(), "UpstreamProxyHosts": cluster.UpstreamProxy.hosts(), "UpstreamProxyNoProxy": cluster.UpstreamProxy.noProxy(), + "HostAliases": cluster.HostAliases, "IsPublic": h.IsPublic, "HiveType": "hosted", "HasGHE": effectiveGitHubBaseURL(h, cluster) != "", @@ -3424,6 +3432,18 @@ spec: app: hive hive-id: {{.ID}} spec: +{{- if .HostAliases}} + # Per-cluster host_aliases (clusters.json): pins GHE hostnames to a TLS + # reverse-proxy IP where no CONNECT proxy exists. See #17801. + hostAliases: +{{- range .HostAliases}} + - ip: "{{.IP}}" + hostnames: +{{- range .Hostnames}} + - "{{.}}" +{{- end}} +{{- end}} +{{- end}} {{- if .RequiresSCC}} serviceAccountName: hive-sa {{- else}} @@ -3464,11 +3484,12 @@ spec: # Steady state (#17468): /etc/hive is an emptyDir, so a regular file # there is a throwaway cache and operator edits to it vanished on # restart. Symlink it to the PVC runtime config instead, so the path - # every runbook names IS the persisted boot source. Config.Save writes - # the config path in place (O_TRUNC, no rename), so the link survives - # saves. Only the NEW name is linked: the legacy .bak is never written - # (config.RuntimeConfigFileLegacy), and linking it would make every - # save write it. + # every runbook names IS the persisted boot source. Config.Save + # resolves the link and renames its temp file onto the TARGET + # (#17862), so the link survives saves. Only the NEW name is linked: + # the legacy .bak is refreshed by the entrypoint from a good runtime + # file on clean boots (never written by Save), and linking it would + # make every save write it. command: ["sh", "-c", "if [ -s /data/hive.yaml.runtime ]; then echo runtime-config-exists-for-recovery; ln -sf /data/hive.yaml.runtime /etc/hive/hive.yaml && echo config-path-linked-to-runtime || echo config-path-link-failed; elif [ -s /data/hive.yaml.bak ]; then echo legacy-runtime-config-exists-for-recovery; else cp /etc/hive-seed/hive.yaml /etc/hive/hive.yaml && echo configmap-copied-first-boot; fi"] volumeMounts: - name: config diff --git a/src/pkg/hub/saas_sha_poller.go b/src/pkg/hub/saas_sha_poller.go index 6c04fc0e9f..be74e30875 100644 --- a/src/pkg/hub/saas_sha_poller.go +++ b/src/pkg/hub/saas_sha_poller.go @@ -715,6 +715,15 @@ func (s *HubServer) StartLatestSHAPoller(ctx context.Context) { // rate-limited to perHiveEnvMaxPatchesPerCycle patches per cycle, because // each patch rolls that hive's pod. See perhive_env_reconcile.go. s.reconcilePerHiveEnvIfDue() + // Converge per-cluster host_aliases (clusters.json) onto existing hive + // Deployments so GHE-behind-reverse-proxy clusters never need a manual + // kubectl patch again (#17801). Throttled and rate-limited like the lane + // above because each patch rolls a pod. + s.reconcileHostAliasesIfDue() + // Converge the template startupProbe (#9692) and current init image repo + // onto hive Deployments provisioned before them, so a slow boot is not a + // permanent liveness kill-loop (#17827). Throttled and rate-limited. + s.reconcileStartupProbeIfDue() // Put #7457's auth-url / auth-response-headers onto the hive-contribute // Ingress of every hosted spoke provisioned before that fix, so // /api/contribute/me learns who is calling there too (#7517). The @@ -789,6 +798,8 @@ func (s *HubServer) pollLatestSHAsTick(ctx context.Context, now time.Time) { s.reconcileNetAdminIfDue() s.reconcileNodeHealthRBACIfDue() s.reconcilePerHiveEnvIfDue() + s.reconcileHostAliasesIfDue() + s.reconcileStartupProbeIfDue() s.reapOrphanedPodsIfDue() s.retireExpiredGenerationsIfDue() s.sweepExpiredAccessIfDue() diff --git a/src/pkg/hub/server.go b/src/pkg/hub/server.go index 0a83f2adf1..7e791ec929 100644 --- a/src/pkg/hub/server.go +++ b/src/pkg/hub/server.go @@ -1220,6 +1220,12 @@ type HubServer struct { // Same rationale and same guarding mutex as lastNetAdminReconcile above: // both are poller-loop-only state. lastPerHiveEnvReconcile time.Time + // lastHostAliasesReconcile throttles the per-cluster host_aliases + // reconcile (cluster_host_aliases.go). Same mutex, same rationale. + lastHostAliasesReconcile time.Time + // lastStartupProbeReconcile throttles the template startupProbe / init + // image reconcile (startup_probe_reconcile.go). Same mutex, same rationale. + lastStartupProbeReconcile time.Time // lastContributeIngressReconcile throttles the hive-contribute Ingress // auth-url reconcile (contribute_ingress_reconcile.go), which puts the // #7457 annotations onto spokes provisioned before that fix (#7517). diff --git a/src/pkg/hub/spoke/cluster_metrics.go b/src/pkg/hub/spoke/cluster_metrics.go index aef70da225..031bc03a0c 100644 --- a/src/pkg/hub/spoke/cluster_metrics.go +++ b/src/pkg/hub/spoke/cluster_metrics.go @@ -14,6 +14,8 @@ import ( "strings" "sync" "time" + + "github.com/hivecommons/hive/crux/egressmark" ) const ( @@ -547,7 +549,7 @@ func k8sAPIGetWithContext(ctx context.Context, path string) ([]byte, error) { client := &http.Client{ Timeout: metricsCollectionTimeout, - Transport: &http.Transport{TLSClientConfig: tlsConfig}, + Transport: &http.Transport{TLSClientConfig: tlsConfig, DialContext: egressmark.Dialer(0).DialContext}, } ctx, cancel := context.WithTimeout(ctx, metricsCollectionTimeout) diff --git a/src/pkg/hub/spoke/heartbeat.go b/src/pkg/hub/spoke/heartbeat.go index 868ae5de12..20ab8c0a49 100644 --- a/src/pkg/hub/spoke/heartbeat.go +++ b/src/pkg/hub/spoke/heartbeat.go @@ -23,6 +23,7 @@ import ( "time" "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/pkg/config" "github.com/hivecommons/hive/pkg/fleetreport" "github.com/hivecommons/hive/pkg/hivestate" @@ -1552,11 +1553,38 @@ func StartHeartbeat(ctx context.Context, hubURL string, collect StatusCollector, logger.Info("hub heartbeat stopped") return case <-ticker.C: + if heartbeatUnprovisionedBackoffActive(time.Now()) { + // Keep the attempt clock advancing so /api/livez does not + // read the deliberate pause as a stalled loop. + recordHeartbeatAttempt() + continue + } processHeartbeatResponse(sendHeartbeat(ctx, hubURL, collect, logger, freshCollect)) } } } +// heartbeatUnprovisionedBackoff is how long the loop pauses after the hub +// answers 403 "hosted/saas hive IDs can only be created via provisioning". +// The hub never auto-creates hosted IDs, so retrying every tick cannot +// succeed until an operator provisions (or the hive is re-IDed); a slow probe +// still notices when that happens. A var so tests can shorten it. +var heartbeatUnprovisionedBackoff = 10 * time.Minute + +// heartbeatUnprovisionedUntil is the UnixNano deadline of that pause (0 = none). +var heartbeatUnprovisionedUntil atomic.Int64 + +func heartbeatUnprovisionedBackoffActive(now time.Time) bool { + until := heartbeatUnprovisionedUntil.Load() + return until != 0 && now.UnixNano() < until +} + +// hubHeartbeatRejectsUnprovisioned reports whether the hub refused the beat +// because it does not know this hosted/saas hive ID (a permanent condition). +func hubHeartbeatRejectsUnprovisioned(status int, body string) bool { + return status == http.StatusForbidden && strings.Contains(body, "can only be created via provisioning") +} + // waitForReady's knobs are vars (not consts) for the same reason as // taskPushInterval: tests need a StartHeartbeat that reaches its send loop in // milliseconds, not after a 3-minute readiness wait. Production keeps the @@ -1867,6 +1895,12 @@ func postHeartbeatToHub(ctx context.Context, hubURL string, payload *HeartbeatPa if resp.StatusCode >= 300 { respBody, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + if hubHeartbeatRejectsUnprovisioned(resp.StatusCode, string(respBody)) { + heartbeatUnprovisionedUntil.Store(time.Now().Add(heartbeatUnprovisionedBackoff).UnixNano()) + logger.Warn("hub heartbeat rejected: hive id is not provisioned on the hub — backing off", + "status", resp.StatusCode, "body", string(respBody), "retry_in", heartbeatUnprovisionedBackoff.String()) + return nil + } logger.Warn("hub heartbeat rejected", "status", resp.StatusCode, "body", string(respBody)) if hubHeartbeatStatusIsDefect(resp.StatusCode) { fleetreport.Observe(fleetreport.ClassUnexpectedHub5xx, "heartbeat", fmt.Sprintf("hub heartbeat returned HTTP %d", resp.StatusCode), string(respBody)) @@ -1874,6 +1908,7 @@ func postHeartbeatToHub(ctx context.Context, hubURL string, payload *HeartbeatPa return nil } fleetreport.Clear(fleetreport.ClassUnexpectedHub5xx, "heartbeat") + heartbeatUnprovisionedUntil.Store(0) // The hub accepted the beat (status < 300) — this is what /api/livez // treats as "the heartbeat goroutine is alive and doing its job", so @@ -2304,6 +2339,7 @@ func inClusterHTTPClient(cfg *inClusterConfig) (*http.Client, error) { Timeout: routeExistenceProbeTimeout, Transport: &http.Transport{ TLSClientConfig: &tls.Config{RootCAs: roots, MinVersion: tls.VersionTLS12}, + DialContext: egressmark.Dialer(0).DialContext, }, }, nil } diff --git a/src/pkg/hub/spoke/heartbeat_activity.go b/src/pkg/hub/spoke/heartbeat_activity.go index 566c4fd5fc..bdcec1e543 100644 --- a/src/pkg/hub/spoke/heartbeat_activity.go +++ b/src/pkg/hub/spoke/heartbeat_activity.go @@ -122,12 +122,39 @@ func HeartbeatKickInterval(govState governor.State, name string, proc *crux.Proc return 0 } cadence, ok := govState.Cadences[name] - if !ok || cadence.Paused || cadence.Interval <= 0 { + if !ok || cadence.Paused { + return 0 + } + if cadence.Schedule != "" && cadence.Schedule.Mode() != config.CadenceModeInterval { + return scheduledKickGap(cadence, govState.LastKick[name], time.Now()) + } + if cadence.Interval <= 0 { return 0 } return cadence.Interval } +// scheduledKickGap is the cadence evidence for a times/cron agent: the gap +// from the slot it last fired at to the slot it fires at next. The hub's +// idle rule compares pane quiet against this (plus slack), so without it a +// "09:00,12:00,15:00 weekdays" agent reads as idle-with-work every evening +// and all weekend — exactly when it is supposed to be quiet. With no fired +// slot yet the gap runs from now to the first slot. Zero means no evidence. +func scheduledKickGap(cadence governor.AgentCadence, lastKick, now time.Time) time.Duration { + base := cadence.LastOccurrence + if base.IsZero() { + base = lastKick + } + if base.IsZero() || base.After(now) { + base = now + } + next, ok := cadence.Schedule.NextAfter(base) + if !ok || !next.After(base) { + return 0 + } + return next.Sub(base) +} + func QuotaExhaustedAgentCount(agents []AgentSummary) int { count := 0 for _, a := range agents { diff --git a/src/pkg/hub/spoke/heartbeat_activity_test.go b/src/pkg/hub/spoke/heartbeat_activity_test.go index fd0e6053cc..eb8100c9a9 100644 --- a/src/pkg/hub/spoke/heartbeat_activity_test.go +++ b/src/pkg/hub/spoke/heartbeat_activity_test.go @@ -102,6 +102,31 @@ func TestHeartbeatKickInterval(t *testing.T) { } } +func TestScheduledKickGapSpansSlots(t *testing.T) { + // Weekday 09:00/12:00/15:00 UTC. Last fired Friday 15:00 → next slot is + // Monday 09:00, so the gap must cover the whole weekend, not 3h. + cad := governor.AgentCadence{Schedule: config.Cadence(`tod:{"times":["09:00","12:00","15:00"],"tz":"UTC","days":["mon","tue","wed","thu","fri"]}`)} + if err := cad.Schedule.Validate(); err != nil { + t.Fatalf("fixture cadence invalid: %v", err) + } + fri := time.Date(2026, 11, 13, 15, 0, 0, 0, time.UTC) + cad.LastOccurrence = fri + if got := scheduledKickGap(cad, fri, fri.Add(5*time.Hour)); got != 66*time.Hour { + t.Errorf("weekend gap = %v, want 66h", got) + } + // Mid-day slot: 09:00 → 12:00. + cad.LastOccurrence = time.Date(2026, 11, 11, 9, 0, 0, 0, time.UTC) + if got := scheduledKickGap(cad, time.Time{}, cad.LastOccurrence.Add(time.Hour)); got != 3*time.Hour { + t.Errorf("intra-day gap = %v, want 3h", got) + } + // Never fired: gap runs from now to the first slot. + cad.LastOccurrence = time.Time{} + now := time.Date(2026, 11, 11, 7, 0, 0, 0, time.UTC) + if got := scheduledKickGap(cad, time.Time{}, now); got != 2*time.Hour { + t.Errorf("unfired gap = %v, want 2h", got) + } +} + func TestQuotaExhaustedCountsAndReasons(t *testing.T) { statuses := map[string]*crux.Process{ "counted": {State: crux.StateRunning, QuotaExhausted: true}, diff --git a/src/pkg/hub/spoke/heartbeat_testhooks.go b/src/pkg/hub/spoke/heartbeat_testhooks.go index 87216c3f17..48743a0619 100644 --- a/src/pkg/hub/spoke/heartbeat_testhooks.go +++ b/src/pkg/hub/spoke/heartbeat_testhooks.go @@ -29,6 +29,7 @@ func SetHeartbeatStateForTest(enabled bool, lastAttempt, lastSuccess time.Time) // still had fresh stats to fall back on. func ResetHeartbeatStateForTest() { SetHeartbeatStateForTest(false, time.Time{}, time.Time{}) + heartbeatUnprovisionedUntil.Store(0) lastGoodPayload.Lock() // Invalidate any collect goroutine still running from an earlier test // (same lock as its late store) so it cannot repopulate the cache. diff --git a/src/pkg/hub/spoke/heartbeat_unprovisioned_test.go b/src/pkg/hub/spoke/heartbeat_unprovisioned_test.go new file mode 100644 index 0000000000..4aa9227c4d --- /dev/null +++ b/src/pkg/hub/spoke/heartbeat_unprovisioned_test.go @@ -0,0 +1,63 @@ +package spoke + +import ( + "context" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/fleetreport" +) + +func TestPostHeartbeat_UnprovisionedForbiddenBacksOff(t *testing.T) { + t.Cleanup(ResetHeartbeatStateForTest) + ResetHeartbeatStateForTest() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "hosted/saas hive IDs can only be created via provisioning", http.StatusForbidden) + })) + defer srv.Close() + + if resp := postHeartbeatToHub(context.Background(), srv.URL, &HeartbeatPayload{}, nil2Logger()); resp != nil { + t.Fatalf("rejected beat returned a response: %+v", resp) + } + if !heartbeatUnprovisionedBackoffActive(time.Now()) { + t.Fatal("expected backoff after unprovisioned 403") + } + if heartbeatUnprovisionedBackoffActive(time.Now().Add(heartbeatUnprovisionedBackoff + time.Second)) { + t.Fatal("backoff must expire") + } +} + +func TestPostHeartbeat_OtherForbiddenDoesNotBackOff(t *testing.T) { + t.Cleanup(ResetHeartbeatStateForTest) + ResetHeartbeatStateForTest() + + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "bad secret", http.StatusForbidden) + })) + defer srv.Close() + + postHeartbeatToHub(context.Background(), srv.URL, &HeartbeatPayload{}, nil2Logger()) + if heartbeatUnprovisionedBackoffActive(time.Now()) { + t.Fatal("a 403 without the provisioning body must not back off") + } +} + +func TestHubHeartbeatRejectsUnprovisioned(t *testing.T) { + body := "hosted/saas hive IDs can only be created via provisioning" + if !hubHeartbeatRejectsUnprovisioned(http.StatusForbidden, body) { + t.Error("403 + body should match") + } + if hubHeartbeatRejectsUnprovisioned(http.StatusUnauthorized, body) { + t.Error("401 must not match") + } +} + +func TestUnprovisionedWarnIsLogFedClass(t *testing.T) { + msg := "hub heartbeat rejected: hive id is not provisioned on the hub — backing off" + if got := fleetreport.LogFedClass(msg); got != fleetreport.ClassHubHeartbeatUnprovisioned { + t.Fatalf("LogFedClass = %q", got) + } +} diff --git a/src/pkg/hub/spoke/self_upgrade.go b/src/pkg/hub/spoke/self_upgrade.go index d39ec3096e..aeb81dc804 100644 --- a/src/pkg/hub/spoke/self_upgrade.go +++ b/src/pkg/hub/spoke/self_upgrade.go @@ -14,6 +14,7 @@ import ( "sync/atomic" "time" + "github.com/hivecommons/hive/crux/egressmark" "github.com/hivecommons/hive/pkg/imageref" ) @@ -762,7 +763,7 @@ func k8sAPIPatchOnce(path string, body []byte) error { client := &http.Client{ Timeout: selfUpgradePatchTimeout, - Transport: &http.Transport{TLSClientConfig: tlsConfig}, + Transport: &http.Transport{TLSClientConfig: tlsConfig, DialContext: egressmark.Dialer(0).DialContext}, } ctx, cancel := context.WithTimeout(context.Background(), selfUpgradePatchTimeout) diff --git a/src/pkg/hub/startup_probe_reconcile.go b/src/pkg/hub/startup_probe_reconcile.go new file mode 100644 index 0000000000..6ef0007df4 --- /dev/null +++ b/src/pkg/hub/startup_probe_reconcile.go @@ -0,0 +1,258 @@ +package hub + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "time" +) + +const ( + // startupProbeReconcileInterval throttles the sweep. Like the NET_ADMIN and + // host_aliases lanes the drift is STATIC: a Deployment either carries the + // template's startupProbe / current init image or it does not. + startupProbeReconcileInterval = 15 * time.Minute + + // startupProbeKubectlTimeout bounds each per-hive kubectl get/patch so one + // unreachable cluster cannot stall the whole sweep. + startupProbeKubectlTimeout = 15 * time.Second + + // startupProbeMaxPatchesPerCycle caps PATCHES per sweep: every patch + // changes the pod template and rolls that hive's pod once. + startupProbeMaxPatchesPerCycle = 3 + + // legacyHiveImageRepo is the pre-rename image repository. Its tags are no + // longer published, so an init container still pinned to it fails to pull + // on the next reschedule (#17827). + legacyHiveImageRepo = "ghcr.io/kubestellar/hive" + + // templateHiveImageRepo is the repository the provisioning template renders + // for every hive container and init container. + templateHiveImageRepo = bulkHiveImageRepo + + // hiveContainerName is the main container's name in the template. + hiveContainerName = "hive" +) + +// templateStartupProbe is the startupProbe the provisioning template renders +// onto the hive container (#9692), as the JSON value a patch writes. +type templateStartupProbe struct { + HTTPGet struct { + Path string `json:"path"` + Port int `json:"port"` + } `json:"httpGet"` + InitialDelaySeconds int `json:"initialDelaySeconds"` + PeriodSeconds int `json:"periodSeconds"` + FailureThreshold int `json:"failureThreshold"` +} + +func wantHostedStartupProbe() templateStartupProbe { + var p templateStartupProbe + p.HTTPGet.Path = "/api/health" + p.HTTPGet.Port = dashboardPort + p.InitialDelaySeconds = hostedStartupProbeInitialDelaySeconds + p.PeriodSeconds = hostedStartupProbePeriodSeconds + p.FailureThreshold = hostedStartupProbeFailureThreshold + return p +} + +// liveProbe is the subset of a live probe the template controls. Fields the +// API server defaults (scheme, timeoutSeconds, successThreshold) are ignored +// so a converged Deployment never compares unequal. +type liveProbe struct { + HTTPGet *struct { + Path string `json:"path"` + Port json.RawMessage `json:"port"` + } `json:"httpGet"` + InitialDelaySeconds int `json:"initialDelaySeconds"` + PeriodSeconds int `json:"periodSeconds"` + FailureThreshold int `json:"failureThreshold"` +} + +type liveContainer struct { + Name string `json:"name"` + Image string `json:"image"` + StartupProbe *liveProbe `json:"startupProbe"` +} + +type liveHiveDeployment struct { + Spec struct { + Template struct { + Spec struct { + InitContainers []liveContainer `json:"initContainers"` + Containers []liveContainer `json:"containers"` + } `json:"spec"` + } `json:"template"` + } `json:"spec"` +} + +func startupProbeMatches(live *liveProbe, want templateStartupProbe) bool { + if live == nil || live.HTTPGet == nil { + return false + } + port := strings.Trim(string(live.HTTPGet.Port), `"`) + return live.HTTPGet.Path == want.HTTPGet.Path && + port == strconv.Itoa(want.HTTPGet.Port) && + live.InitialDelaySeconds == want.InitialDelaySeconds && + live.PeriodSeconds == want.PeriodSeconds && + live.FailureThreshold == want.FailureThreshold +} + +// rewriteLegacyHiveImage returns the image with a legacyHiveImageRepo +// repository swapped for templateHiveImageRepo, keeping the tag (or digest). +// ok is false when the image is not on the legacy repository. +func rewriteLegacyHiveImage(image string) (string, bool) { + image = strings.TrimSpace(image) + if !strings.HasPrefix(image, legacyHiveImageRepo) { + return "", false + } + rest := image[len(legacyHiveImageRepo):] + if rest != "" && rest[0] != ':' && rest[0] != '@' { + return "", false + } + return templateHiveImageRepo + rest, true +} + +// startupProbePatchJSON is the PURE decision function for the sweep. Given the +// live Deployment JSON (`kubectl get deployment hive -o json`), it returns the +// JSON-Patch that sets the hive container's startupProbe to the template's and +// rewrites any init-container image on the legacy repository, plus a short +// description of each change. An empty patch means already converged. +func startupProbePatchJSON(deployJSON []byte) (string, []string, error) { + var d liveHiveDeployment + if err := json.Unmarshal(deployJSON, &d); err != nil { + return "", nil, fmt.Errorf("parsing deployment: %w", err) + } + podSpec := d.Spec.Template.Spec + var ops []map[string]any + var changes []string + + idx := -1 + for i, c := range podSpec.Containers { + if c.Name == hiveContainerName { + idx = i + break + } + } + if idx >= 0 { + want := wantHostedStartupProbe() + if !startupProbeMatches(podSpec.Containers[idx].StartupProbe, want) { + ops = append(ops, map[string]any{ + "op": "add", + "path": fmt.Sprintf("/spec/template/spec/containers/%d/startupProbe", idx), + "value": want, + }) + changes = append(changes, "startupProbe") + } + } + + for i, c := range podSpec.InitContainers { + img, ok := rewriteLegacyHiveImage(c.Image) + if !ok { + continue + } + ops = append(ops, map[string]any{ + "op": "replace", + "path": fmt.Sprintf("/spec/template/spec/initContainers/%d/image", i), + "value": img, + }) + changes = append(changes, "initContainers["+c.Name+"].image="+img) + } + + if len(ops) == 0 { + return "", nil, nil + } + b, err := json.Marshal(ops) + if err != nil { + return "", nil, err + } + return string(b), changes, nil +} + +// reconcileStartupProbeIfDue runs the sweep at most once per +// startupProbeReconcileInterval. Safe to call from the poller loop every tick; +// same guarding mutex as reconcileHostAliasesIfDue (poller-loop-only state). +func (s *HubServer) reconcileStartupProbeIfDue() { + s.clusterUnreachableMu.Lock() + due := s.lastStartupProbeReconcile.IsZero() || + time.Since(s.lastStartupProbeReconcile) >= startupProbeReconcileInterval + if due { + s.lastStartupProbeReconcile = time.Now() + } + s.clusterUnreachableMu.Unlock() + if !due { + return + } + s.reconcileStartupProbe() +} + +// reconcileStartupProbe converges the template's startupProbe (#9692) and the +// current init-container image repository onto hosted hive Deployments +// provisioned before those landed (#17827). Without the startupProbe a slow +// boot is killed by the livenessProbe at ~90s and never recovers. At most +// startupProbeMaxPatchesPerCycle patches per cycle, since each rolls a pod. +func (s *HubServer) reconcileStartupProbe() { + patched := 0 + deferred := 0 + for _, h := range listSaaSHives() { + if !netAdminSweepEligible(h.Status) { + continue + } + cluster := s.clusterForHive(&h) + if cluster == nil || s.clusterRecentlyUnreachable(cluster.ID) { + continue + } + ns := hiveHostedNamespacePrefix + h.ID + + ctx, cancel := context.WithTimeout(context.Background(), startupProbeKubectlTimeout) + getCmd := kubectlForClusterContext(ctx, cluster, "get", "deployment", "hive", + "-n", ns, "-o", "json") + out, err := getCmd.Output() + cancel() + if err != nil { + s.logger.Debug("startupProbe reconcile: could not read hive deployment", + "hive_id", h.ID, "cluster", cluster.ID, "error", err) + continue + } + s.markClusterReachable(cluster.ID) + + patch, changes, perr := startupProbePatchJSON(out) + if perr != nil { + s.logger.Warn("startupProbe reconcile: could not build patch", + "hive_id", h.ID, "cluster", cluster.ID, "error", perr) + continue + } + if patch == "" { + continue + } + if patched >= startupProbeMaxPatchesPerCycle { + deferred++ + continue + } + pctx, pcancel := context.WithTimeout(context.Background(), startupProbeKubectlTimeout) + patchCmd := kubectlForClusterContext(pctx, cluster, "patch", "deployment", "hive", + "-n", ns, "--type=json", "-p", patch) + pout, err := patchCmd.CombinedOutput() + pcancel() + if err != nil { + s.markClusterUnreachable(cluster.ID) + s.logger.Warn("startupProbe reconcile: patch failed — will retry next sweep", + "hive_id", h.ID, "cluster", cluster.ID, + "output", strings.TrimSpace(string(pout)), "error", err) + continue + } + s.markClusterReachable(cluster.ID) + patched++ + s.logger.Info("reconciled template startupProbe/init image onto hive deployment (rolls the pod once)", + "hive_id", h.ID, "cluster", cluster.ID, + "changes", strings.Join(changes, ","), "patched_this_cycle", patched) + } + if deferred > 0 { + s.logger.Info("startupProbe reconcile: rate limit reached, remaining hives deferred to next cycle", + "patched", patched, "deferred", deferred, + "max_per_cycle", startupProbeMaxPatchesPerCycle, + "next_cycle_in", startupProbeReconcileInterval.String()) + } +} diff --git a/src/pkg/hub/startup_probe_reconcile_test.go b/src/pkg/hub/startup_probe_reconcile_test.go new file mode 100644 index 0000000000..a250e9af05 --- /dev/null +++ b/src/pkg/hub/startup_probe_reconcile_test.go @@ -0,0 +1,136 @@ +package hub + +import ( + "encoding/json" + "strings" + "testing" +) + +type testPatchOp struct { + Op string `json:"op"` + Path string `json:"path"` + Value json.RawMessage `json:"value"` +} + +func decodeStartupProbePatch(t *testing.T, patch string) []testPatchOp { + t.Helper() + var ops []testPatchOp + if err := json.Unmarshal([]byte(patch), &ops); err != nil { + t.Fatalf("patch is not valid JSON-Patch: %v\n%s", err, patch) + } + return ops +} + +func TestStartupProbePatchAddsMissingProbe(t *testing.T) { + live := []byte(`{"spec":{"template":{"spec":{ + "initContainers":[{"name":"copy-config","image":"ghcr.io/hivecommons/hive:v4-latest"}], + "containers":[ + {"name":"sidecar","image":"busybox"}, + {"name":"hive","image":"ghcr.io/hivecommons/hive:v4-latest", + "livenessProbe":{"httpGet":{"path":"/api/livez","port":3002},"periodSeconds":30,"failureThreshold":3}} + ]}}}}`) + patch, changes, err := startupProbePatchJSON(live) + if err != nil { + t.Fatal(err) + } + ops := decodeStartupProbePatch(t, patch) + if len(ops) != 1 || ops[0].Op != "add" || ops[0].Path != "/spec/template/spec/containers/1/startupProbe" { + t.Fatalf("unexpected patch: %s", patch) + } + var got templateStartupProbe + if err := json.Unmarshal(ops[0].Value, &got); err != nil { + t.Fatal(err) + } + if got != wantHostedStartupProbe() { + t.Fatalf("patched probe %+v != template %+v", got, wantHostedStartupProbe()) + } + if got.HTTPGet.Path != "/api/health" || got.HTTPGet.Port != dashboardPort || got.FailureThreshold != hostedStartupProbeFailureThreshold { + t.Fatalf("probe does not match the template: %+v", got) + } + if len(changes) != 1 || changes[0] != "startupProbe" { + t.Fatalf("unexpected changes: %v", changes) + } +} + +func TestStartupProbePatchReplacesDriftedProbe(t *testing.T) { + live := []byte(`{"spec":{"template":{"spec":{"containers":[{"name":"hive", + "startupProbe":{"httpGet":{"path":"/api/health","port":3002},"initialDelaySeconds":10,"periodSeconds":5,"failureThreshold":30}}]}}}}`) + patch, _, err := startupProbePatchJSON(live) + if err != nil { + t.Fatal(err) + } + if ops := decodeStartupProbePatch(t, patch); len(ops) != 1 || ops[0].Path != "/spec/template/spec/containers/0/startupProbe" { + t.Fatalf("drifted probe must be replaced: %s", patch) + } +} + +func TestStartupProbePatchNoopWhenConverged(t *testing.T) { + want := wantHostedStartupProbe() + // API-server defaulted fields (scheme, timeoutSeconds, successThreshold) + // must not make a converged Deployment look drifted. + probe := map[string]any{ + "httpGet": map[string]any{"path": want.HTTPGet.Path, "port": want.HTTPGet.Port, "scheme": "HTTP"}, + "initialDelaySeconds": want.InitialDelaySeconds, + "periodSeconds": want.PeriodSeconds, + "failureThreshold": want.FailureThreshold, + "timeoutSeconds": 1, + "successThreshold": 1, + } + dep := map[string]any{"spec": map[string]any{"template": map[string]any{"spec": map[string]any{ + "initContainers": []any{map[string]any{"name": "init-permissions", "image": "ghcr.io/hivecommons/hive:v4-latest"}}, + "containers": []any{map[string]any{"name": "hive", "startupProbe": probe}}, + }}}} + live, _ := json.Marshal(dep) + patch, changes, err := startupProbePatchJSON(live) + if err != nil { + t.Fatal(err) + } + if patch != "" || len(changes) != 0 { + t.Fatalf("converged deployment must not be patched, got %s (%v)", patch, changes) + } +} + +func TestStartupProbePatchRewritesLegacyInitImageKeepingTag(t *testing.T) { + want, _ := json.Marshal(wantHostedStartupProbe()) + live := []byte(`{"spec":{"template":{"spec":{ + "initContainers":[ + {"name":"copy-config","image":"ghcr.io/hivecommons/hive:v4-latest"}, + {"name":"init-permissions","image":"ghcr.io/kubestellar/hive:v2-latest"} + ], + "containers":[{"name":"hive","startupProbe":` + string(want) + `}]}}}}`) + patch, changes, err := startupProbePatchJSON(live) + if err != nil { + t.Fatal(err) + } + ops := decodeStartupProbePatch(t, patch) + if len(ops) != 1 || ops[0].Op != "replace" || ops[0].Path != "/spec/template/spec/initContainers/1/image" { + t.Fatalf("unexpected patch: %s", patch) + } + var img string + if err := json.Unmarshal(ops[0].Value, &img); err != nil { + t.Fatal(err) + } + if img != "ghcr.io/hivecommons/hive:v2-latest" { + t.Fatalf("legacy init image must move to the template repo keeping its tag, got %q", img) + } + if len(changes) != 1 || !strings.Contains(changes[0], "init-permissions") { + t.Fatalf("unexpected changes: %v", changes) + } +} + +func TestRewriteLegacyHiveImage(t *testing.T) { + for in, want := range map[string]string{ + "ghcr.io/kubestellar/hive:v2-latest": "ghcr.io/hivecommons/hive:v2-latest", + "ghcr.io/kubestellar/hive": "ghcr.io/hivecommons/hive", + "ghcr.io/kubestellar/hive@sha256:abc": "ghcr.io/hivecommons/hive@sha256:abc", + } { + if got, ok := rewriteLegacyHiveImage(in); !ok || got != want { + t.Errorf("%s: got %q,%v want %q", in, got, ok, want) + } + } + for _, in := range []string{"ghcr.io/hivecommons/hive:v4-latest", "ghcr.io/kubestellar/hive-other:v1", "busybox"} { + if _, ok := rewriteLegacyHiveImage(in); ok { + t.Errorf("%s must not be rewritten", in) + } + } +} diff --git a/src/pkg/inbox/dibs.go b/src/pkg/inbox/dibs.go new file mode 100644 index 0000000000..334c2c203b --- /dev/null +++ b/src/pkg/inbox/dibs.go @@ -0,0 +1,113 @@ +package inbox + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" +) + +// dibsTimeout bounds one idea submission. +const dibsTimeout = 30 * time.Second + +// dibsMaxBody caps the idea body so a long transcript does not trip dibs' +// request limit; the full text stays in Knowledge. +const dibsMaxBody = 60_000 + +// DibsDestination posts an item as a private draft idea to a dibs instance +// (`POST {url}/api/ideas`, bearer hub token). +type DibsDestination struct { + URL string + Token string + HTTP *http.Client +} + +// Name implements Destination. +func (d *DibsDestination) Name() string { return DestDibs } + +// ErrDibsUnconfigured is returned when inbox.dibs.url is empty. +var ErrDibsUnconfigured = errors.New("dibs: inbox.dibs.url is not configured") + +type dibsIdea struct { + Title string `json:"title"` + Body string `json:"body"` + Visibility string `json:"visibility"` + Status string `json:"status"` +} + +// Deliver implements Destination. +func (d *DibsDestination) Deliver(ctx context.Context, it Item, c Content, _ Request) (Result, error) { + base := strings.TrimRight(strings.TrimSpace(d.URL), "/") + if base == "" { + return Result{}, ErrDibsUnconfigured + } + if strings.TrimSpace(d.Token) == "" { + return Result{}, errors.New("dibs: no token (set inbox.dibs.token_env to a variable holding a hub bearer token)") + } + body := DibsBody(it, c) + payload, _ := json.Marshal(dibsIdea{ + Title: firstNonEmpty(c.Title, it.Title), + Body: body, + Visibility: "private", + Status: "draft", + }) + rctx, cancel := context.WithTimeout(ctx, dibsTimeout) + defer cancel() + req, err := http.NewRequestWithContext(rctx, http.MethodPost, base+"/api/ideas", bytes.NewReader(payload)) + if err != nil { + return Result{}, err + } + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "Bearer "+d.Token) + client := d.HTTP + if client == nil { + client = http.DefaultClient + } + resp, err := client.Do(req) + if err != nil { + return Result{}, fmt.Errorf("dibs: %w", err) + } + defer resp.Body.Close() + data, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20)) + if resp.StatusCode < 200 || resp.StatusCode > 299 { + return Result{}, fmt.Errorf("dibs: %s: %s", resp.Status, lastLines(string(data), 2)) + } + var idea struct { + ID string `json:"id"` + } + if err := json.Unmarshal(data, &idea); err != nil || idea.ID == "" { + return Result{}, fmt.Errorf("dibs: response has no idea id: %s", lastLines(string(data), 2)) + } + return Result{URL: base + "/api/ideas/" + idea.ID}, nil +} + +// DibsBody renders the idea body: summary, provenance, then transcript. +func DibsBody(it Item, c Content) string { + var b strings.Builder + if c.Summary != "" { + b.WriteString(c.Summary) + b.WriteString("\n\n") + } + fmt.Fprintf(&b, "_Source: %s_\n", c.Provenance) + if !it.RecordedAt.IsZero() { + fmt.Fprintf(&b, "_Recorded: %s_\n", it.RecordedAt.UTC().Format(time.RFC3339)) + } + text := c.Transcript + if text == "" && c.Summary == "" { + text = c.Markdown + } + if text != "" { + b.WriteString("\n## Transcript\n\n") + b.WriteString(text) + } + out := b.String() + if len(out) > dibsMaxBody { + out = out[:dibsMaxBody] + "\n\n_(truncated; full transcript in Knowledge)_" + } + return out +} diff --git a/src/pkg/inbox/edges_test.go b/src/pkg/inbox/edges_test.go new file mode 100644 index 0000000000..52ea44841a --- /dev/null +++ b/src/pkg/inbox/edges_test.go @@ -0,0 +1,276 @@ +package inbox + +import ( + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +func TestServiceSettingsFindAndConfigUpdate(t *testing.T) { + src := &fakeSource{items: []Item{{Source: "plaud", ID: "rec-1", Title: "one"}}} + spek := &recordingDest{name: DestSpekRFC} + dibs := &recordingDest{name: DestDibs} + svc := newTestService(t, config.InboxConfig{}, src, spek, dibs) + if spek.Name() != DestSpekRFC || dibs.Name() != DestDibs { + t.Fatal("destination names") + } + + st := svc.Settings() + if st.Rules == nil || len(st.Rules) != 0 || st.SpekReady || st.DibsTokenSet { + t.Fatalf("default settings = %+v", st) + } + svc.SetReadiness(func() bool { return true }, func() bool { return true }) + svc.UpdateConfig(config.InboxConfig{ + Rules: []config.InboxRule{{TitlePrefix: "RFC:", Destination: DestSpekRFC}}, + Spek: config.InboxSpek{Repo: "o/r"}, + Dibs: config.InboxDibs{URL: "https://dibs.example"}, + }) + st = svc.Settings() + if !st.SpekReady || !st.DibsTokenSet || st.SpekRepo != "o/r" || st.DibsURL != "https://dibs.example" || len(st.Rules) != 1 { + t.Fatalf("settings after update = %+v", st) + } + + it, ok, err := svc.Find(context.Background(), "plaud", "rec-1") + if err != nil || !ok || it.Title != "one" { + t.Fatalf("find = %+v %v %v", it, ok, err) + } + if _, ok, _ := svc.Find(context.Background(), "plaud", "missing"); ok { + t.Fatal("found a missing item") + } + svc.Wait() +} + +func TestServiceNowAndSortItems(t *testing.T) { + fixed := time.Date(2026, 3, 4, 5, 6, 7, 0, time.FixedZone("x", 3600)) + svc := newTestService(t, config.InboxConfig{}, &fakeSource{}, &recordingDest{name: DestSpekRFC}) + if !svc.now().Before(time.Now().Add(time.Second)) { + t.Fatal("now() without override should be wall clock") + } + svc.Now = func() time.Time { return fixed } + if got := svc.now(); got != fixed.UTC() { + t.Fatalf("now = %v", got) + } + items := []Item{ + {ID: "a", RecordedAt: fixed.Add(-time.Hour)}, + {ID: "b", RecordedAt: fixed}, + {ID: "c", RecordedAt: fixed.Add(-2 * time.Hour)}, + } + sortItems(items) + if items[0].ID != "b" || items[1].ID != "a" || items[2].ID != "c" { + t.Fatalf("sorted = %v %v %v", items[0].ID, items[1].ID, items[2].ID) + } +} + +type failingSource struct { + fakeSource + listErr error + fetchErr error +} + +func (f *failingSource) List(context.Context) ([]Item, error) { return f.items, f.listErr } +func (f *failingSource) Fetch(context.Context, Item) (Content, error) { + return f.body, f.fetchErr +} + +func TestServiceDeliverFailures(t *testing.T) { + src := &failingSource{fakeSource: fakeSource{items: []Item{{Source: "plaud", ID: "rec-1", Title: "one"}}}, fetchErr: errors.New("boom")} + dest := &recordingDest{name: DestSpekRFC, res: Result{Note: "delegated to agent x"}} + svc := newTestService(t, config.InboxConfig{Spek: config.InboxSpek{Repo: "o/r"}}, src, dest) + + // Fetch error is recorded as failed. + items, _ := svc.Items(context.Background()) + if _, err := svc.Route(context.Background(), items[0], Request{Destination: DestSpekRFC}); err != nil { + t.Fatal(err) + } + items, _ = svc.Items(context.Background()) + if r := items[0].Routes[DestSpekRFC]; r == nil || r.Status != StatusFailed || !strings.Contains(r.Error, "boom") { + t.Fatalf("fetch failure = %+v", r) + } + + // Unknown source is recorded as failed. + ghost := Item{Source: "ghost", ID: "g-1"} + if _, err := svc.Route(context.Background(), ghost, Request{Destination: DestSpekRFC}); err != nil { + t.Fatal(err) + } + if r, ok := svc.Store.Route(ghost.Key(), DestSpekRFC); !ok || r.Status != StatusFailed || !strings.Contains(r.Error, "not configured") { + t.Fatalf("ghost route = %+v %v", r, ok) + } + + // A note-only result (no URL) becomes delegated. + src.fetchErr = nil + if _, err := svc.Route(context.Background(), items[0], Request{Destination: DestSpekRFC}); err != nil { + t.Fatal(err) + } + if r, ok := svc.Store.Route(items[0].Key(), DestSpekRFC); !ok || r.Status != StatusDelegated || r.Error != "delegated to agent x" { + t.Fatalf("delegated route = %+v", r) + } + + // Items surfaces list errors but still returns items. + src.listErr = errors.New("list down") + items, err := svc.Items(context.Background()) + if err == nil || !strings.Contains(err.Error(), "list down") || len(items) != 1 { + t.Fatalf("items with list error = %d %v", len(items), err) + } + if _, err := svc.Sweep(context.Background()); err == nil { + t.Fatal("sweep should surface the list error") + } + if known := svc.Store.Known(); len(known) != 2 || known[0] != ghost.Key() { + t.Fatalf("known = %v", known) + } +} + +func TestStoreEdgeCases(t *testing.T) { + fixed := time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC) + mem := NewStore("") + if mem.now().IsZero() { + t.Fatal("now") + } + mem.Now = func() time.Time { return fixed } + if st, err := mem.Begin("plaud/x", DestDibs, ""); err != nil || st.UpdatedAt != fixed { + t.Fatalf("in-memory begin = %+v %v", st, err) + } + if _, ok := mem.Route("plaud/x", DestSpekRFC); ok { + t.Fatal("route for another destination should be absent") + } + if _, ok := mem.Route("nope", DestDibs); ok { + t.Fatal("route for unknown key should be absent") + } + + // Corrupt file is tolerated; unwritable dir surfaces an error. + dir := t.TempDir() + bad := filepath.Join(dir, "bad.json") + if err := os.WriteFile(bad, []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + if known := NewStore(bad).Known(); len(known) != 0 { + t.Fatalf("corrupt store known = %v", known) + } + blocker := filepath.Join(dir, "file") + if err := os.WriteFile(blocker, nil, 0o600); err != nil { + t.Fatal(err) + } + if _, err := NewStore(filepath.Join(blocker, "inbox.json")).Begin("k", DestDibs, ""); err == nil { + t.Fatal("save under a file should fail") + } +} + +func TestFactSourceEdgeCases(t *testing.T) { + src := &FactSource{ConnectorsDir: filepath.Join(t.TempDir(), "missing"), SourceType: "wisprflow"} + if src.Type() != "wisprflow" { + t.Fatal("type") + } + if items, err := src.List(context.Background()); err != nil || items != nil { + t.Fatalf("missing dir = %v %v", items, err) + } + + dir := t.TempDir() + src.ConnectorsDir = dir + if err := os.WriteFile(filepath.Join(dir, "stray.txt"), nil, 0o600); err != nil { + t.Fatal(err) + } + writeFact(t, dir, "personal", "wisprflow-w-1.md", strings.Replace(plaudFact, "source: plaud", "source: wisprflow", 1)) + writeFact(t, dir, "personal", "wisprflow-w-2.md", "no front matter at all") + writeFact(t, dir, "personal", "wisprflow-w-3.md", "---\nsource: wisprflow\nunterminated") + writeFact(t, dir, "personal", "wisprflow-noid.md", "---\nsource: wisprflow\n---\nbody") + if err := os.Mkdir(filepath.Join(dir, "personal", "wisprflow-dir.md"), 0o755); err != nil { + t.Fatal(err) + } + items, err := src.List(context.Background()) + if err != nil || len(items) != 1 || items[0].Source != "wisprflow" { + t.Fatalf("items = %+v %v", items, err) + } + + ctx, cancel := context.WithCancel(context.Background()) + cancel() + if _, err := src.List(ctx); !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled list = %v", err) + } + + if _, err := src.Fetch(context.Background(), Item{FactPath: filepath.Join(dir, "nope.md")}); err == nil { + t.Fatal("fetch of a missing fact should fail") + } + // Fetch falls back to the item title when the fact has none. + path := filepath.Join(dir, "personal", "wisprflow-untitled.md") + if err := os.WriteFile(path, []byte("---\nsource: wisprflow\nsource_id: u\n---\nhello"), 0o600); err != nil { + t.Fatal(err) + } + c, err := src.Fetch(context.Background(), Item{ID: "u", Title: "from item", FactPath: path}) + if err != nil || c.Title != "from item" || c.Markdown != "hello" || c.Provenance != "wisprflow://recordings/u" { + t.Fatalf("fetch = %+v %v", c, err) + } + + fm, body := splitFrontMatter("---\na: 1\n---") + if fm["a"] != "1" || body != "" { + t.Fatalf("front matter without trailing newline = %v %q", fm, body) + } +} + +func TestSpekDestinationRealExecAndLookPath(t *testing.T) { + d := &SpekDestination{Binary: "definitely-not-a-binary-" + t.Name(), DefaultRepo: "o/r"} + it := Item{Source: "plaud", ID: "r", Title: "RFC: x"} + _, err := d.Deliver(context.Background(), it, Content{Transcript: "hi"}, Request{}) + if err == nil || !strings.Contains(err.Error(), "not installed") { + t.Fatalf("missing binary = %v", err) + } + if d.lookPath("sh") != nil { + t.Skip("sh not on PATH") + } + out, err := d.exec(context.Background(), "sh", []string{"-c", "cat; echo done"}, os.Environ(), "input\n") + if err != nil || !strings.Contains(out, "input") || !strings.Contains(out, "done") { + t.Fatalf("exec = %q %v", out, err) + } + if got := lastLines("a\nb\nc\n", 2); got != "b\nc" { + t.Fatalf("lastLines = %q", got) + } + if got := (&SpekDestination{Prefixes: []string{"X:"}}).prefixes(); len(got) != 1 || got[0] != "X:" { + t.Fatalf("prefixes = %v", got) + } +} + +func TestSpekDestinationErrorPaths(t *testing.T) { + it := Item{Source: "plaud", ID: "r", Title: "RFC: x"} + ok := func(string) error { return nil } + cases := []struct { + name string + d *SpekDestination + c Content + want string + }{ + {"no repo", &SpekDestination{LookPath: ok}, Content{Transcript: "t"}, ErrNoRepo.Error()}, + {"empty", &SpekDestination{LookPath: ok, DefaultRepo: "o/r"}, Content{}, "no transcript"}, + {"token", &SpekDestination{LookPath: ok, DefaultRepo: "o/r", Token: func(context.Context, string) (string, error) { return "", errors.New("mint") }}, Content{Transcript: "t"}, "github token: mint"}, + {"exec", &SpekDestination{LookPath: ok, DefaultRepo: "o/r", Exec: func(context.Context, string, []string, []string, string) (string, error) { + return "l1\nl2\nboom", errors.New("exit 2") + }}, Content{Transcript: "t"}, "boom"}, + {"parse", &SpekDestination{LookPath: ok, DefaultRepo: "o/r", Exec: func(context.Context, string, []string, []string, string) (string, error) { + return "nothing useful", nil + }}, Content{Transcript: "t"}, "spek_rfc:"}, + {"kick fails", &SpekDestination{LookPath: func(string) error { return errors.New("nope") }, DefaultRepo: "o/r", Agent: "a", Kick: func(string, string) error { return errors.New("kick") }}, Content{Transcript: "t"}, "kick to a failed"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + _, err := tc.d.Deliver(context.Background(), it, tc.c, Request{}) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Fatalf("err = %v, want %q", err, tc.want) + } + }) + } +} + +func TestNewServiceDefaultsLogger(t *testing.T) { + svc := NewService(config.InboxConfig{}, NewStore(""), nil, nil, nil) + if svc.Logger == nil { + t.Fatal("nil logger should default") + } + svc.Logger = slog.New(slog.NewTextHandler(os.Stderr, nil)) + if items, err := svc.Items(context.Background()); err != nil || len(items) != 0 { + t.Fatalf("items with no sources = %v %v", items, err) + } +} diff --git a/src/pkg/inbox/factsource.go b/src/pkg/inbox/factsource.go new file mode 100644 index 0000000000..5785290aac --- /dev/null +++ b/src/pkg/inbox/factsource.go @@ -0,0 +1,209 @@ +package inbox + +import ( + "context" + "os" + "path/filepath" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" +) + +// excerptRunes bounds the list excerpt. +const excerptRunes = 240 + +// FactSource lists items of one connector type by reading the facts that +// type's connectors wrote under <ConnectorsDir>/<layer>/. It needs nothing +// from the vendor: any connector whose facts carry `source`/`source_id` +// front matter is an Inbox source. +type FactSource struct { + // ConnectorsDir is <knowledge base>/connectors. + ConnectorsDir string + // SourceType is the connector type (plaud, wisprflow, ...). + SourceType string +} + +// Type implements Source. +func (f *FactSource) Type() string { return f.SourceType } + +// List implements Source: newest first. +func (f *FactSource) List(ctx context.Context) ([]Item, error) { + layers, err := os.ReadDir(f.ConnectorsDir) + if os.IsNotExist(err) { + return nil, nil + } + if err != nil { + return nil, err + } + var out []Item + for _, layer := range layers { + if !layer.IsDir() { + continue + } + dir := filepath.Join(f.ConnectorsDir, layer.Name()) + files, err := os.ReadDir(dir) + if err != nil { + continue + } + for _, fi := range files { + if ctx.Err() != nil { + return out, ctx.Err() + } + if fi.IsDir() || !strings.HasSuffix(fi.Name(), ".md") || !strings.HasPrefix(fi.Name(), f.SourceType+"-") { + continue + } + path := filepath.Join(dir, fi.Name()) + data, err := os.ReadFile(path) + if err != nil { + continue + } + fm, body := splitFrontMatter(string(data)) + if fm["source"] != f.SourceType || fm["source_id"] == "" || fm["status"] == "deprecated" { + continue + } + out = append(out, itemFromFact(fm, body, layer.Name(), path)) + } + } + sort.SliceStable(out, func(i, j int) bool { + if !out[i].RecordedAt.Equal(out[j].RecordedAt) { + return out[i].RecordedAt.After(out[j].RecordedAt) + } + return out[i].ID < out[j].ID + }) + return out, nil +} + +func itemFromFact(fm map[string]string, body, layer, path string) Item { + it := Item{ + Source: fm["source"], + Connector: fm["connector"], + ID: fm["source_id"], + Title: fm["title"], + FactSlug: strings.TrimSuffix(filepath.Base(path), ".md"), + FactPath: path, + Layer: layer, + Excerpt: excerpt(body), + } + if it.Title == "" { + it.Title = it.ID + } + for _, key := range []string{"attr_recorded_at", "synthesized"} { + if t, err := time.Parse(time.RFC3339, fm[key]); err == nil { + it.RecordedAt = t.UTC() + break + } + } + if ms, err := strconv.ParseInt(fm["attr_duration_ms"], 10, 64); err == nil { + it.DurationMS = ms + } + if tags := fm["attr_tags"]; tags != "" { + for _, t := range strings.Split(tags, ",") { + if t = strings.TrimSpace(t); t != "" { + it.Tags = append(it.Tags, t) + } + } + } + return it +} + +// Fetch implements Source by re-reading the fact and splitting the body +// into the connector's "## Summary" / "## Transcript" sections. +func (f *FactSource) Fetch(_ context.Context, it Item) (Content, error) { + data, err := os.ReadFile(it.FactPath) + if err != nil { + return Content{}, err + } + fm, body := splitFrontMatter(string(data)) + c := Content{ + Title: fm["title"], + Markdown: strings.TrimSpace(body), + Provenance: f.SourceType + "://recordings/" + it.ID, + } + if c.Title == "" { + c.Title = it.Title + } + c.Summary, c.Transcript = splitSections(body) + return c, nil +} + +// splitFrontMatter mirrors the connector fact layout: a `---` block of +// `key: value` lines, then the markdown body. +func splitFrontMatter(content string) (map[string]string, string) { + out := map[string]string{} + if !strings.HasPrefix(content, "---\n") { + return out, content + } + end := strings.Index(content[4:], "\n---") + if end < 0 { + return out, content + } + for _, l := range strings.Split(content[4:4+end], "\n") { + k, v, ok := strings.Cut(l, ":") + if !ok { + continue + } + out[strings.TrimSpace(k)] = strings.Trim(strings.TrimSpace(v), "\"'") + } + rest := content[4+end+4:] + if i := strings.Index(rest, "\n"); i >= 0 { + rest = rest[i+1:] + } else { + rest = "" + } + return out, rest +} + +// splitSections returns the text under a "## Summary…" heading and under +// "## Transcript". Headings may carry a suffix such as "(Plaud)". +func splitSections(body string) (summary, transcript string) { + var cur *string + var sb, tb strings.Builder + for _, line := range strings.Split(body, "\n") { + if strings.HasPrefix(line, "## ") { + h := strings.ToLower(line[3:]) + switch { + case strings.HasPrefix(h, "summary"): + cur = &summary + case strings.HasPrefix(h, "transcript"): + cur = &transcript + default: + cur = nil + } + continue + } + if cur == &summary { + sb.WriteString(line) + sb.WriteByte('\n') + } else if cur == &transcript { + tb.WriteString(line) + tb.WriteByte('\n') + } + } + return strings.TrimSpace(sb.String()), strings.TrimSpace(tb.String()) +} + +// excerpt returns the first prose lines of body, bounded to excerptRunes. +func excerpt(body string) string { + var b strings.Builder + for _, line := range strings.Split(body, "\n") { + line = strings.TrimSpace(line) + if line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "**") { + continue + } + if b.Len() > 0 { + b.WriteByte(' ') + } + b.WriteString(line) + if utf8.RuneCountInString(b.String()) >= excerptRunes { + break + } + } + s := b.String() + if utf8.RuneCountInString(s) > excerptRunes { + r := []rune(s) + s = strings.TrimSpace(string(r[:excerptRunes])) + "…" + } + return s +} diff --git a/src/pkg/inbox/inbox.go b/src/pkg/inbox/inbox.go new file mode 100644 index 0000000000..bc6e194e16 --- /dev/null +++ b/src/pkg/inbox/inbox.go @@ -0,0 +1,123 @@ +// Package inbox turns voice-recorder items that knowledge connectors already +// ingest as facts (Plaud, Wispr Flow, ...) into a source-agnostic triage list +// with routes to spek RFCs, dibs ideas, or "knowledge only". +package inbox + +import ( + "context" + "time" +) + +// Route statuses for one (item, destination) pair. +const ( + StatusQueued = "queued" + StatusRunning = "running" + StatusFiled = "filed" + StatusFailed = "failed" +) + +// Destination names. Knowledge is implicit: items are Knowledge facts. +const ( + DestSpekRFC = "spek_rfc" + DestDibs = "dibs" + DestDismiss = "dismiss" +) + +// Item is one incoming recording/dictation as the Inbox lists it. +type Item struct { + // Source is the connector type (plaud, wisprflow, ...). + Source string `json:"source"` + // Connector is the knowledge.connectors[] entry name that synced it. + Connector string `json:"connector"` + // ID is the upstream id (Plaud file id, ...). (Source, ID) is the key. + ID string `json:"id"` + Title string `json:"title"` + Excerpt string `json:"excerpt,omitempty"` + RecordedAt time.Time `json:"recorded_at,omitempty"` + DurationMS int64 `json:"duration_ms,omitempty"` + Tags []string `json:"tags,omitempty"` + // FactSlug / FactPath / Layer locate the Knowledge fact backing the item. + FactSlug string `json:"fact_slug"` + FactPath string `json:"fact_path,omitempty"` + Layer string `json:"layer,omitempty"` + // Routes holds per-destination state; absent means never routed. + Routes map[string]*RouteState `json:"routes,omitempty"` + // Dismissed marks "knowledge only": the item stays a fact, no further route. + Dismissed bool `json:"dismissed,omitempty"` + // Processed is true once any route filed or the item was dismissed. + Processed bool `json:"processed"` +} + +// Key returns the stable "source/id" key used by the state store. +func (it Item) Key() string { return Key(it.Source, it.ID) } + +// Key joins a source and item id. +func Key(source, id string) string { return source + "/" + id } + +// RouteState is the lifecycle of one item on one destination. +type RouteState struct { + Status string `json:"status"` + // Repo is the owner/repo a spek_rfc route targeted. + Repo string `json:"repo,omitempty"` + // URL is the RFC issue or dibs idea once filed. + URL string `json:"url,omitempty"` + // Children are the RFC's child issue URLs. + Children []string `json:"children,omitempty"` + Error string `json:"error,omitempty"` + UpdatedAt time.Time `json:"updated_at"` +} + +// Content is the full item body a destination needs. +type Content struct { + Title string + Summary string + Transcript string + // Markdown is the whole fact body when Summary/Transcript are not split. + Markdown string + // Provenance is a stable reference such as plaud://recordings/<id>. + Provenance string +} + +// Text returns transcript+summary, falling back to the raw markdown. +func (c Content) Text() string { + if c.Transcript == "" && c.Summary == "" { + return c.Markdown + } + out := c.Transcript + if c.Summary != "" { + if out != "" { + out += "\n\n" + } + out += "Summary:\n" + c.Summary + } + return out +} + +// Source lists and fetches items of one type. The fact-backed source covers +// every connector type; vendor-specific sources can replace it later. +type Source interface { + Type() string + List(ctx context.Context) ([]Item, error) + Fetch(ctx context.Context, it Item) (Content, error) +} + +// Request is one routing request. +type Request struct { + Destination string + // Repo is the owner/repo for spek_rfc; empty uses the configured default. + Repo string +} + +// Result is what a destination filed. +type Result struct { + URL string + Children []string + // Note is set instead of URL when the work was delegated to an agent. + Note string +} + +// Destination files an item somewhere outside Knowledge. +type Destination interface { + Name() string + Deliver(ctx context.Context, it Item, c Content, req Request) (Result, error) +} diff --git a/src/pkg/inbox/inbox_test.go b/src/pkg/inbox/inbox_test.go new file mode 100644 index 0000000000..34868adcaa --- /dev/null +++ b/src/pkg/inbox/inbox_test.go @@ -0,0 +1,398 @@ +package inbox + +import ( + "context" + "encoding/json" + "errors" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +const plaudFact = `--- +title: RFC: Voice inbox +type: reference +layer: personal +status: active +tags: [connector, plaud, plaud] +source: plaud +connector: plaud +source_id: rec-1 +source_path: Plaud +attr_duration_ms: 125000 +attr_plaud_file_id: rec-1 +attr_recorded_at: 2026-11-10T08:58:00Z +attr_tags: voice, ideas +synthesized: 2026-11-10T08:58:00Z +synced_at: 2026-11-11T20:00:00Z +--- + +# RFC: Voice inbox + +**Recorded:** 2026-11-10T08:58:00Z + +## Summary (Plaud) + +- ship the inbox + +## Transcript + +[00:02] Andy: Hello hive +[01:05] Andy: Second line +` + +func writeFact(t *testing.T, dir, layer, name, content string) string { + t.Helper() + p := filepath.Join(dir, layer, name) + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(p, []byte(content), 0o644); err != nil { + t.Fatal(err) + } + return p +} + +func TestFactSourceListsAndFetchesConnectorFacts(t *testing.T) { + dir := t.TempDir() + writeFact(t, dir, "personal", "plaud-plaud-rec-1.md", plaudFact) + writeFact(t, dir, "personal", "plaud-plaud-rec-0.md", strings.Replace(strings.Replace(plaudFact, "rec-1", "rec-0", -1), "status: active", "status: deprecated", 1)) + writeFact(t, dir, "personal", "notion-docs-x.md", strings.Replace(plaudFact, "source: plaud", "source: notion", 1)) + src := &FactSource{ConnectorsDir: dir, SourceType: "plaud"} + items, err := src.List(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(items) != 1 { + t.Fatalf("items = %+v", items) + } + it := items[0] + if it.ID != "rec-1" || it.Source != "plaud" || it.Connector != "plaud" || it.Title != "RFC: Voice inbox" || it.FactSlug != "plaud-plaud-rec-1" || it.Layer != "personal" { + t.Fatalf("item = %+v", it) + } + if it.DurationMS != 125000 || !it.RecordedAt.Equal(time.Date(2026, 11, 10, 8, 58, 0, 0, time.UTC)) || strings.Join(it.Tags, ",") != "voice,ideas" { + t.Fatalf("attrs = %+v", it) + } + if !strings.Contains(it.Excerpt, "ship the inbox") || strings.Contains(it.Excerpt, "Recorded:") { + t.Fatalf("excerpt = %q", it.Excerpt) + } + c, err := src.Fetch(context.Background(), it) + if err != nil { + t.Fatal(err) + } + if c.Summary != "- ship the inbox" || !strings.HasPrefix(c.Transcript, "[00:02] Andy: Hello hive") || c.Provenance != "plaud://recordings/rec-1" { + t.Fatalf("content = %+v", c) + } +} + +func TestRuleMatching(t *testing.T) { + it := Item{Source: "plaud", Title: "rfc: Build the thing", Tags: []string{"Ideas"}} + cases := []struct { + name string + rule Rule + want bool + }{ + {"prefix case-insensitive", Rule{TitlePrefix: "RFC:"}, true}, + {"prefix mismatch", Rule{TitlePrefix: "Hive:"}, false}, + {"tag", Rule{Tag: "ideas"}, true}, + {"source mismatch", Rule{Source: "wisprflow", TitlePrefix: "RFC:"}, false}, + {"source match", Rule{Source: "plaud", TitlePrefix: "RFC:"}, true}, + {"no filter never matches", Rule{Destination: DestDibs}, false}, + } + for _, tc := range cases { + if got := RuleMatches(tc.rule, it); got != tc.want { + t.Errorf("%s: got %v want %v", tc.name, got, tc.want) + } + } + if got := StripPrefix("RFC: Build the thing", DefaultTitlePrefixes); got != "Build the thing" { + t.Fatalf("StripPrefix = %q", got) + } + if got := StripPrefix("Plain", DefaultTitlePrefixes); got != "Plain" { + t.Fatalf("StripPrefix = %q", got) + } +} + +func TestStoreTransitions(t *testing.T) { + st := NewStore(filepath.Join(t.TempDir(), "inbox.json")) + key := Key("plaud", "rec-1") + if _, err := st.Begin(key, DestSpekRFC, "o/r"); err != nil { + t.Fatal(err) + } + if _, err := st.Begin(key, DestSpekRFC, "o/r"); !errors.Is(err, ErrRouteBusy) { + t.Fatalf("second Begin while queued: %v", err) + } + if _, err := st.Set(key, DestSpekRFC, StatusRunning, Result{}, ""); err != nil { + t.Fatal(err) + } + if _, err := st.Begin(key, DestSpekRFC, "o/r"); !errors.Is(err, ErrRouteBusy) { + t.Fatalf("Begin while running: %v", err) + } + // dibs route on the same item is independent. + if _, err := st.Begin(key, DestDibs, ""); err != nil { + t.Fatal(err) + } + if _, err := st.Set(key, DestSpekRFC, StatusFiled, Result{URL: "https://github.com/o/r/issues/1", Children: []string{"https://github.com/o/r/issues/2"}}, ""); err != nil { + t.Fatal(err) + } + if _, err := st.Set(key, DestDibs, StatusFailed, Result{}, "boom"); err != nil { + t.Fatal(err) + } + // Reload from disk. + st2 := NewStore(st.Path) + items := []Item{{Source: "plaud", ID: "rec-1"}, {Source: "plaud", ID: "rec-9"}} + st2.Apply(items) + r := items[0].Routes[DestSpekRFC] + if r == nil || r.Status != StatusFiled || r.URL != "https://github.com/o/r/issues/1" || len(r.Children) != 1 || r.Repo != "o/r" { + t.Fatalf("spek route = %+v", r) + } + if d := items[0].Routes[DestDibs]; d == nil || d.Status != StatusFailed || d.Error != "boom" { + t.Fatalf("dibs route = %+v", d) + } + if !items[0].Processed || items[1].Processed { + t.Fatalf("processed flags = %v %v", items[0].Processed, items[1].Processed) + } + // A failed route may be retried; a filed one may be re-filed. + if _, err := st2.Begin(key, DestDibs, ""); err != nil { + t.Fatalf("retry after failure: %v", err) + } + if err := st2.Dismiss(Key("plaud", "rec-9"), true); err != nil { + t.Fatal(err) + } + st2.Apply(items) + if !items[1].Processed || !items[1].Dismissed { + t.Fatalf("dismissed item must be processed: %+v", items[1]) + } +} + +func TestParsePublishOutput(t *testing.T) { + jsonOut := `Publishing... +{"target":"github","repo":"o/r","rfc_url":"https://github.com/o/r/issues/10","rfc_number":10,"children":[{"id":"a","url":"https://github.com/o/r/issues/11","number":11},{"id":"b","url":"https://github.com/o/r/issues/12","number":12}],"created":3}` + res, err := ParsePublishOutput(jsonOut) + if err != nil { + t.Fatal(err) + } + if res.URL != "https://github.com/o/r/issues/10" || len(res.Children) != 2 || res.Children[1] != "https://github.com/o/r/issues/12" { + t.Fatalf("json result = %+v", res) + } + textOut := "Draft written to .spektacular/specs/voice.md\nRFC: https://github.com/o/r/issues/20\n child: https://github.com/o/r/issues/21\n child: https://github.com/o/r/issues/22 (dup) https://github.com/o/r/issues/22\n" + res, err = ParsePublishOutput(textOut) + if err != nil { + t.Fatal(err) + } + if res.URL != "https://github.com/o/r/issues/20" || len(res.Children) != 2 { + t.Fatalf("text result = %+v", res) + } + if _, err := ParsePublishOutput("error: nothing happened"); err == nil { + t.Fatal("no URL must be an error") + } +} + +func TestRenderAgentPrompt(t *testing.T) { + it := Item{Source: "plaud", ID: "rec-1"} + c := Content{Summary: "- ship it", Transcript: "Hello hive", Provenance: "plaud://recordings/rec-1"} + p := RenderAgentPrompt(it, c, "o/r", "Voice inbox v2") + for _, want := range []string{"spektacular publish voice-inbox-v2 --repo o/r --allow-draft --format json", "spektacular init", "Open questions", "plaud://recordings/rec-1", "## Summary\n- ship it", "## Transcript\nHello hive"} { + if !strings.Contains(p, want) { + t.Errorf("prompt missing %q:\n%s", want, p) + } + } +} + +type fakeSource struct { + items []Item + body Content +} + +func (f *fakeSource) Type() string { return "plaud" } +func (f *fakeSource) List(context.Context) ([]Item, error) { return f.items, nil } +func (f *fakeSource) Fetch(context.Context, Item) (Content, error) { return f.body, nil } + +func newTestService(t *testing.T, cfg config.InboxConfig, src Source, dests ...Destination) *Service { + t.Helper() + store := NewStore(filepath.Join(t.TempDir(), "inbox.json")) + svc := NewService(cfg, store, []Source{src}, dests, slog.New(slog.NewTextHandler(os.Stderr, nil))) + svc.Run = func(fn func()) { fn() } + return svc +} + +func TestSpekDestinationRunsVoicePublish(t *testing.T) { + var gotArgs []string + var gotStdin string + var gotEnv []string + calls := 0 + d := &SpekDestination{ + Binary: "spektacular", + DefaultRepo: "o/r", + Token: func(context.Context, string) (string, error) { return "tok", nil }, + LookPath: func(string) error { return nil }, + Exec: func(_ context.Context, _ string, args, env []string, stdin string) (string, error) { + calls++ + if calls == 1 { + return "Error: unknown flag: --format", errors.New("exit status 1") + } + gotArgs, gotEnv, gotStdin = args, env, stdin + return "RFC: https://github.com/o/r/issues/5\nhttps://github.com/o/r/issues/6\n", nil + }, + } + it := Item{Source: "plaud", ID: "rec-1", Title: "RFC: Voice inbox"} + c := Content{Title: "RFC: Voice inbox", Transcript: "Hello hive", Summary: "- ship it", Provenance: "plaud://recordings/rec-1"} + res, err := d.Deliver(context.Background(), it, c, Request{Destination: DestSpekRFC}) + if err != nil { + t.Fatal(err) + } + if calls != 2 { + t.Fatalf("expected retry without --format json, calls=%d", calls) + } + want := []string{"voice", "--from", "stdin", "--title", "Voice inbox", "--publish", "--repo", "o/r"} + if strings.Join(gotArgs, " ") != strings.Join(want, " ") { + t.Fatalf("args = %v", gotArgs) + } + if !strings.Contains(gotStdin, "Hello hive") || !strings.Contains(gotStdin, "plaud://recordings/rec-1") { + t.Fatalf("stdin = %q", gotStdin) + } + hasTok := false + for _, e := range gotEnv { + if e == "GITHUB_TOKEN=tok" { + hasTok = true + } + } + if !hasTok { + t.Fatal("GITHUB_TOKEN not injected") + } + if res.URL != "https://github.com/o/r/issues/5" || len(res.Children) != 1 { + t.Fatalf("result = %+v", res) + } +} + +func TestSpekDestinationFallsBackToAgentKick(t *testing.T) { + var kicked, prompt string + d := &SpekDestination{ + LookPath: func(string) error { return errors.New("not found") }, + Agent: "builder", + Kick: func(agent, p string) error { kicked, prompt = agent, p; return nil }, + } + res, err := d.Deliver(context.Background(), Item{Source: "plaud", ID: "x", Title: "Idea"}, Content{Markdown: "stuff"}, Request{Repo: "o/r"}) + if err != nil { + t.Fatal(err) + } + if kicked != "builder" || !strings.Contains(prompt, "--repo o/r") || res.Note == "" || res.URL != "" { + t.Fatalf("kick=%q note=%q prompt=%q", kicked, res.Note, prompt) + } + d.Agent = "" + if _, err := d.Deliver(context.Background(), Item{Source: "plaud", ID: "x"}, Content{Markdown: "stuff"}, Request{Repo: "o/r"}); err == nil { + t.Fatal("missing binary without agent must fail") + } + if _, err := (&SpekDestination{LookPath: func(string) error { return nil }}).Deliver(context.Background(), Item{}, Content{Markdown: "x"}, Request{}); !errors.Is(err, ErrNoRepo) { + t.Fatalf("no repo: %v", err) + } +} + +type recordingDest struct { + name string + mu sync.Mutex + got []Request + res Result + err error +} + +func (r *recordingDest) Name() string { return r.name } +func (r *recordingDest) Deliver(_ context.Context, _ Item, _ Content, req Request) (Result, error) { + r.mu.Lock() + defer r.mu.Unlock() + r.got = append(r.got, req) + return r.res, r.err +} + +func TestServiceRouteAndSweep(t *testing.T) { + src := &fakeSource{items: []Item{ + {Source: "plaud", ID: "rec-1", Title: "RFC: one", RecordedAt: time.Date(2026, 1, 2, 0, 0, 0, 0, time.UTC)}, + {Source: "plaud", ID: "rec-2", Title: "note", RecordedAt: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)}, + }, body: Content{Markdown: "x"}} + spek := &recordingDest{name: DestSpekRFC, res: Result{URL: "https://github.com/o/r/issues/1"}} + dibs := &recordingDest{name: DestDibs, err: errors.New("dibs down")} + cfg := config.InboxConfig{ + Rules: []config.InboxRule{{TitlePrefix: "RFC:", Destination: DestSpekRFC}}, + Spek: config.InboxSpek{Repo: "o/r"}, + } + svc := newTestService(t, cfg, src, spek, dibs) + + routed, err := svc.Sweep(context.Background()) + if err != nil || routed != 1 { + t.Fatalf("sweep routed=%d err=%v", routed, err) + } + if len(spek.got) != 1 || spek.got[0].Repo != "o/r" { + t.Fatalf("spek requests = %+v", spek.got) + } + items, _ := svc.Items(context.Background()) + if items[0].ID != "rec-1" || items[0].Routes[DestSpekRFC].Status != StatusFiled || !items[0].Processed { + t.Fatalf("rec-1 = %+v", items[0]) + } + if items[1].Processed || Unprocessed(items) != 1 { + t.Fatalf("rec-2 should be unprocessed: %+v", items[1]) + } + // A second sweep must not re-fire on seen items. + if routed, _ := svc.Sweep(context.Background()); routed != 0 { + t.Fatalf("second sweep routed %d", routed) + } + // Manual dibs route fails and records the error. + if _, err := svc.Route(context.Background(), items[1], Request{Destination: DestDibs}); err != nil { + t.Fatal(err) + } + items, _ = svc.Items(context.Background()) + if r := items[1].Routes[DestDibs]; r == nil || r.Status != StatusFailed || r.Error != "dibs down" { + t.Fatalf("dibs route = %+v", r) + } + if _, err := svc.Route(context.Background(), items[1], Request{Destination: "nope"}); !errors.Is(err, ErrUnknownDestination) { + t.Fatalf("unknown destination: %v", err) + } + // Dismiss = knowledge only. + if _, err := svc.Route(context.Background(), items[1], Request{Destination: DestDismiss}); err != nil { + t.Fatal(err) + } + items, _ = svc.Items(context.Background()) + if !items[1].Dismissed || Unprocessed(items) != 0 { + t.Fatalf("dismiss failed: %+v", items[1]) + } +} + +func TestDibsDestinationPostsIdea(t *testing.T) { + var gotAuth string + var gotBody map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/api/ideas" || r.Method != http.MethodPost { + t.Errorf("unexpected %s %s", r.Method, r.URL.Path) + } + gotAuth = r.Header.Get("Authorization") + _ = json.NewDecoder(r.Body).Decode(&gotBody) + w.WriteHeader(http.StatusCreated) + _, _ = w.Write([]byte(`{"id":"idea-7","title":"x"}`)) + })) + defer srv.Close() + d := &DibsDestination{URL: srv.URL + "/", Token: "hub-tok"} + it := Item{Source: "plaud", ID: "rec-1", Title: "Idea", RecordedAt: time.Date(2026, 1, 1, 0, 0, 0, 0, time.UTC)} + res, err := d.Deliver(context.Background(), it, Content{Title: "Idea", Summary: "sum", Transcript: "words", Provenance: "plaud://recordings/rec-1"}, Request{}) + if err != nil { + t.Fatal(err) + } + if gotAuth != "Bearer hub-tok" || gotBody["title"] != "Idea" || gotBody["visibility"] != "private" || gotBody["status"] != "draft" { + t.Fatalf("auth=%q body=%v", gotAuth, gotBody) + } + body, _ := gotBody["body"].(string) + if !strings.Contains(body, "plaud://recordings/rec-1") || !strings.Contains(body, "## Transcript") || !strings.HasPrefix(body, "sum") { + t.Fatalf("body = %q", body) + } + if res.URL != srv.URL+"/api/ideas/idea-7" { + t.Fatalf("url = %q", res.URL) + } + if _, err := (&DibsDestination{}).Deliver(context.Background(), it, Content{}, Request{}); !errors.Is(err, ErrDibsUnconfigured) { + t.Fatalf("unconfigured: %v", err) + } +} diff --git a/src/pkg/inbox/plaudsource.go b/src/pkg/inbox/plaudsource.go new file mode 100644 index 0000000000..2443d1b293 --- /dev/null +++ b/src/pkg/inbox/plaudsource.go @@ -0,0 +1,207 @@ +package inbox + +import ( + "context" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +// PlaudType is the Source type for Plaud recordings. +const PlaudType = connector.TypePlaud + +// PlaudSource reads recordings straight from the Plaud connector. Item.ID is +// the Plaud file id (the `attr_plaud_file_id` of the fact the same sync +// writes), so route state carries over between this source and FactSource. +type PlaudSource struct { + // Name is the knowledge.connectors[] entry name. + Name string + Reader connector.PlaudReader + // Ready reports whether the connector is signed in; nil means always. + Ready func(ctx context.Context) bool +} + +// Type implements Source. +func (p *PlaudSource) Type() string { return PlaudType } + +// Available reports whether the connector can be read right now. +func (p *PlaudSource) Available(ctx context.Context) bool { + return p.Reader != nil && (p.Ready == nil || p.Ready(ctx)) +} + +// List implements Source: newest first. +func (p *PlaudSource) List(ctx context.Context) ([]Item, error) { + recs, err := p.Reader.ListRecordings(ctx) + if err != nil { + return nil, err + } + out := make([]Item, 0, len(recs)) + for _, r := range recs { + out = append(out, Item{ + Source: PlaudType, + Connector: p.Name, + ID: r.ID, + Title: r.Title, + RecordedAt: r.RecordedAt, + DurationMS: r.DurationMS, + Tags: r.Tags, + }) + } + sortItems(out) + return out, nil +} + +// Fetch implements Source. +func (p *PlaudSource) Fetch(ctx context.Context, it Item) (Content, error) { + c, err := p.Reader.ReadRecording(ctx, it.ID) + if err != nil { + return Content{}, err + } + var md strings.Builder + md.WriteString("# " + it.Title + "\n") + if c.Summary != "" { + md.WriteString("\n## Summary\n\n" + c.Summary + "\n") + } + if c.Transcript != "" { + md.WriteString("\n## Transcript\n\n" + c.Transcript + "\n") + } + return Content{ + Title: it.Title, + Summary: c.Summary, + Transcript: c.Transcript, + Markdown: strings.TrimSpace(md.String()), + Provenance: PlaudType + "://recordings/" + it.ID, + }, nil +} + +// directListTTL bounds how long a direct Plaud listing is reused before it +// is refreshed in the background. Listing spawns the Plaud MCP process, so +// the dashboard poll must never pay for it. +const directListTTL = 5 * time.Minute + +// directListTimeout bounds one background direct listing. +const directListTimeout = 2 * time.Minute + +// PreferDirect lists through Direct while it is available and healthy and +// falls back to Facts otherwise, so the Inbox keeps working before Plaud is +// signed in or when the MCP server is unreachable. +// +// The direct listing is cached for TTL and refreshed in the background: +// List returns whatever is cached (or just facts while nothing is) and +// never blocks on the Plaud process. Direct items are merged with the facts +// of the same id so they keep FactSlug/FactPath/Layer/Excerpt. +type PreferDirect struct { + Direct *PlaudSource + Facts Source + // TTL overrides directListTTL; zero means the default. + TTL time.Duration + + mu sync.Mutex + direct []Item + fetched time.Time + refreshing bool +} + +// Type implements Source. +func (s *PreferDirect) Type() string { return s.Facts.Type() } + +// List implements Source. +func (s *PreferDirect) List(ctx context.Context) ([]Item, error) { + facts, err := s.Facts.List(ctx) + if !s.Direct.Available(ctx) { + return facts, err + } + direct, stale := s.cached() + if stale { + s.refreshAsync() + } + if direct == nil { + return facts, err + } + return mergeDirect(direct, facts), err +} + +// cached returns the cached direct listing (nil when none or when the last +// refresh failed) and whether it is due for a refresh. +func (s *PreferDirect) cached() (items []Item, stale bool) { + s.mu.Lock() + defer s.mu.Unlock() + ttl := s.TTL + if ttl <= 0 { + ttl = directListTTL + } + return s.direct, !s.refreshing && time.Since(s.fetched) >= ttl +} + +// refreshAsync starts one background Refresh unless one is already running. +func (s *PreferDirect) refreshAsync() { + s.mu.Lock() + if s.refreshing { + s.mu.Unlock() + return + } + s.refreshing = true + s.mu.Unlock() + go func() { + ctx, cancel := context.WithTimeout(context.Background(), directListTimeout) + defer cancel() + _ = s.Refresh(ctx) + }() +} + +// Refresh lists Direct now and caches the result. A failure clears the +// cache so List falls back to Facts, and is not retried before TTL. +func (s *PreferDirect) Refresh(ctx context.Context) error { + items, err := s.Direct.List(ctx) + s.mu.Lock() + defer s.mu.Unlock() + s.refreshing = false + s.fetched = time.Now() + if err != nil { + s.direct = nil + return err + } + if items == nil { + items = []Item{} + } + s.direct = items + return nil +} + +// mergeDirect overlays the fact fields onto direct items with the same id +// and appends facts Plaud no longer lists (outside its since_days window). +func mergeDirect(direct, facts []Item) []Item { + byID := make(map[string]Item, len(facts)) + for _, f := range facts { + byID[f.ID] = f + } + out := make([]Item, 0, len(direct)+len(facts)) + for _, it := range direct { + if f, ok := byID[it.ID]; ok { + it.FactSlug, it.FactPath, it.Layer, it.Excerpt = f.FactSlug, f.FactPath, f.Layer, f.Excerpt + if it.Title == "" { + it.Title = f.Title + } + delete(byID, it.ID) + } + out = append(out, it) + } + for _, f := range facts { + if _, ok := byID[f.ID]; ok { + out = append(out, f) + } + } + sortItems(out) + return out +} + +// Fetch implements Source: fact-backed items (FactPath set) read their fact, +// direct items read Plaud. +func (s *PreferDirect) Fetch(ctx context.Context, it Item) (Content, error) { + if it.FactPath != "" { + return s.Facts.Fetch(ctx, it) + } + return s.Direct.Fetch(ctx, it) +} diff --git a/src/pkg/inbox/plaudsource_test.go b/src/pkg/inbox/plaudsource_test.go new file mode 100644 index 0000000000..7fb9953e2a --- /dev/null +++ b/src/pkg/inbox/plaudsource_test.go @@ -0,0 +1,165 @@ +package inbox + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/hivecommons/hive/internal/testutil" + "github.com/hivecommons/hive/pkg/knowledge/connector" +) + +type fakePlaud struct { + recs []connector.PlaudRecording + content connector.PlaudContent + listErr error + reads []string +} + +func (f *fakePlaud) ListRecordings(context.Context) ([]connector.PlaudRecording, error) { + return f.recs, f.listErr +} + +func (f *fakePlaud) ReadRecording(_ context.Context, id string) (connector.PlaudContent, error) { + f.reads = append(f.reads, id) + return f.content, nil +} + +func TestPlaudSourceListAndFetch(t *testing.T) { + older := time.Date(2026, 11, 10, 8, 0, 0, 0, time.UTC) + newer := older.Add(24 * time.Hour) + fp := &fakePlaud{ + recs: []connector.PlaudRecording{ + {ID: "rec-1", Title: "Old", RecordedAt: older, DurationMS: 1000}, + {ID: "rec-2", Title: "New", RecordedAt: newer, Tags: []string{"voice"}}, + }, + content: connector.PlaudContent{Summary: "sum", Transcript: "words"}, + } + src := &PlaudSource{Name: "my-plaud", Reader: fp} + if src.Type() != "plaud" { + t.Fatalf("type = %q", src.Type()) + } + items, err := src.List(context.Background()) + if err != nil || len(items) != 2 { + t.Fatalf("items=%v err=%v", items, err) + } + if items[0].ID != "rec-2" || items[0].Connector != "my-plaud" || items[0].Source != "plaud" || items[0].Key() != "plaud/rec-2" { + t.Fatalf("first item = %+v", items[0]) + } + if items[1].DurationMS != 1000 || !items[0].RecordedAt.Equal(newer) { + t.Fatalf("mapping lost: %+v", items) + } + c, err := src.Fetch(context.Background(), items[0]) + if err != nil { + t.Fatal(err) + } + if c.Title != "New" || c.Summary != "sum" || c.Transcript != "words" || c.Provenance != "plaud://recordings/rec-2" { + t.Fatalf("content = %+v", c) + } + if len(fp.reads) != 1 || fp.reads[0] != "rec-2" { + t.Fatalf("reads = %v", fp.reads) + } +} + +func TestPreferDirectSelection(t *testing.T) { + dir := t.TempDir() + writeFact(t, dir, "personal", "plaud-plaud-rec-1.md", plaudFact) + facts := &FactSource{ConnectorsDir: dir, SourceType: "plaud"} + fp := &fakePlaud{recs: []connector.PlaudRecording{{ID: "direct-1", Title: "Direct", RecordedAt: time.Date(2026, 11, 11, 8, 0, 0, 0, time.UTC)}}} + ready := false + direct := &PlaudSource{Name: "plaud", Reader: fp, Ready: func(context.Context) bool { return ready }} + src := &PreferDirect{Direct: direct, Facts: facts, TTL: time.Hour} + ctx := context.Background() + + items, err := src.List(ctx) + if err != nil || len(items) != 1 || items[0].ID != "rec-1" { + t.Fatalf("not signed in should use facts: %v %v", items, err) + } + if _, err := src.Fetch(ctx, items[0]); err != nil || len(fp.reads) != 0 { + t.Fatalf("fact item must not hit plaud: err=%v reads=%v", err, fp.reads) + } + + ready = true + if err := src.Refresh(ctx); err != nil { + t.Fatal(err) + } + items, err = src.List(ctx) + if err != nil || len(items) != 2 || items[0].ID != "direct-1" || items[1].ID != "rec-1" { + t.Fatalf("signed in should merge plaud with facts: %v %v", items, err) + } + if items[1].FactPath == "" || items[1].Excerpt == "" { + t.Fatalf("fact-only item lost fact fields: %+v", items[1]) + } + if _, err := src.Fetch(ctx, items[0]); err != nil || len(fp.reads) != 1 { + t.Fatalf("direct item should read plaud: err=%v reads=%v", err, fp.reads) + } + + fp.listErr = errors.New("mcp down") + if err := src.Refresh(ctx); err == nil { + t.Fatal("refresh should report the plaud error") + } + items, err = src.List(ctx) + if err != nil || len(items) != 1 || items[0].ID != "rec-1" { + t.Fatalf("plaud failure should fall back to facts: %v %v", items, err) + } +} + +// List must never pay for the Plaud process: the first signed-in call +// returns facts and kicks off one background refresh. +func TestPreferDirectListDoesNotBlock(t *testing.T) { + dir := t.TempDir() + writeFact(t, dir, "personal", "plaud-plaud-rec-1.md", plaudFact) + facts := &FactSource{ConnectorsDir: dir, SourceType: "plaud"} + fp := &countingPlaud{recs: []connector.PlaudRecording{{ID: "rec-1", Title: "Direct title"}}, gate: make(chan struct{})} + src := &PreferDirect{Direct: &PlaudSource{Name: "plaud", Reader: fp}, Facts: facts, TTL: time.Hour} + ctx := context.Background() + + for i := 0; i < 2; i++ { + items, err := src.List(ctx) + if err != nil || len(items) != 1 || items[0].Title != "RFC: Voice inbox" { + t.Fatalf("list %d while plaud is slow should serve facts: %v %v", i, items, err) + } + } + fp.release() + items := testutil.EventuallyValue(t, 5*time.Second, func() ([]Item, bool) { + got, _ := src.List(ctx) + return got, len(got) == 1 && got[0].Title == "Direct title" + }, "background refresh never landed") + if n := fp.calls.Load(); n != 1 { + t.Fatalf("ListRecordings calls = %d, want 1 (single-flight, cached for TTL)", n) + } + it := items[0] + if it.FactSlug != "plaud-plaud-rec-1" || it.Layer != "personal" || it.FactPath == "" || it.Excerpt == "" { + t.Fatalf("direct item lost fact fields: %+v", it) + } +} + +type countingPlaud struct { + recs []connector.PlaudRecording + calls atomic.Int32 + gate chan struct{} + once sync.Once +} + +func (c *countingPlaud) release() { + c.once.Do(func() { + if c.gate != nil { + close(c.gate) + } + }) +} + +func (c *countingPlaud) ListRecordings(context.Context) ([]connector.PlaudRecording, error) { + c.calls.Add(1) + if c.gate != nil { + <-c.gate + } + return c.recs, nil +} + +func (c *countingPlaud) ReadRecording(context.Context, string) (connector.PlaudContent, error) { + return connector.PlaudContent{}, nil +} diff --git a/src/pkg/inbox/rules.go b/src/pkg/inbox/rules.go new file mode 100644 index 0000000000..61adf1a198 --- /dev/null +++ b/src/pkg/inbox/rules.go @@ -0,0 +1,62 @@ +package inbox + +import ( + "strings" + + "github.com/hivecommons/hive/pkg/config" +) + +// DefaultTitlePrefixes are the prefixes the UI offers when a rule has none. +var DefaultTitlePrefixes = []string{"RFC:", "Hive:"} + +// Rule is one auto-routing rule (mirrors config.InboxRule). +type Rule = config.InboxRule + +// RuleMatches reports whether r applies to it: source (when set) must equal +// the item source, and at least one of title_prefix / tag must match. +// A rule with neither filter never matches, so a bad config cannot route +// every recording. +func RuleMatches(r Rule, it Item) bool { + if src := strings.TrimSpace(r.Source); src != "" && !strings.EqualFold(src, it.Source) { + return false + } + prefix := strings.TrimSpace(r.TitlePrefix) + tag := strings.TrimSpace(r.Tag) + if prefix == "" && tag == "" { + return false + } + if prefix != "" && strings.HasPrefix(strings.ToLower(strings.TrimSpace(it.Title)), strings.ToLower(prefix)) { + return true + } + if tag != "" { + for _, t := range it.Tags { + if strings.EqualFold(strings.TrimSpace(t), tag) { + return true + } + } + } + return false +} + +// FirstRule returns the first rule matching it. +func FirstRule(rules []Rule, it Item) (Rule, bool) { + for _, r := range rules { + if RuleMatches(r, it) { + return r, true + } + } + return Rule{}, false +} + +// StripPrefix removes a matched routing prefix ("RFC: Foo" → "Foo") so the +// filed title reads naturally; unmatched titles are returned trimmed. +func StripPrefix(title string, prefixes []string) string { + t := strings.TrimSpace(title) + for _, p := range prefixes { + p = strings.TrimSpace(p) + if p != "" && len(t) > len(p) && strings.EqualFold(t[:len(p)], p) { + return strings.TrimSpace(t[len(p):]) + } + } + return t +} diff --git a/src/pkg/inbox/service.go b/src/pkg/inbox/service.go new file mode 100644 index 0000000000..ff1bcc3c92 --- /dev/null +++ b/src/pkg/inbox/service.go @@ -0,0 +1,279 @@ +package inbox + +import ( + "context" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + "github.com/hivecommons/hive/pkg/config" +) + +// routeTimeout bounds one background delivery. +const routeTimeout = 15 * time.Minute + +// Settings is the operator-editable part of the Inbox (mirrors inbox.* +// YAML minus binary/agent plumbing). +type Settings struct { + Sources []string `json:"sources"` + Rules []config.InboxRule `json:"rules"` + SpekRepo string `json:"spek_repo"` + DibsURL string `json:"dibs_url"` + // DibsTokenSet reports whether inbox.dibs.token_env resolves; the token + // itself is never returned. + DibsTokenSet bool `json:"dibs_token_set"` + SpekReady bool `json:"spek_ready"` +} + +// Service lists items across sources, applies persisted state, runs +// auto-routing rules and executes manual routes in the background. +type Service struct { + Store *Store + Logger *slog.Logger + // Now is injectable for tests. + Now func() time.Time + + mu sync.RWMutex + cfg config.InboxConfig + sources map[string]Source + dests map[string]Destination + wg sync.WaitGroup + // Run executes a delivery; nil runs it in a goroutine. Tests set it to + // run inline. + Run func(fn func()) + // Status snapshots for the settings view. + spekReady func() bool + dibsTokenSet func() bool +} + +// NewService builds a service for cfg with the given sources/destinations. +func NewService(cfg config.InboxConfig, store *Store, sources []Source, dests []Destination, logger *slog.Logger) *Service { + if logger == nil { + logger = slog.Default() + } + s := &Service{Store: store, Logger: logger, cfg: cfg, sources: map[string]Source{}, dests: map[string]Destination{}} + for _, src := range sources { + s.sources[src.Type()] = src + } + for _, d := range dests { + s.dests[d.Name()] = d + } + return s +} + +// SetReadiness installs the probes behind Settings.SpekReady / DibsTokenSet. +func (s *Service) SetReadiness(spek, dibs func() bool) { + s.spekReady, s.dibsTokenSet = spek, dibs +} + +// Config returns the current inbox config. +func (s *Service) Config() config.InboxConfig { + s.mu.RLock() + defer s.mu.RUnlock() + return s.cfg +} + +// UpdateConfig replaces the routing config (after the dashboard saved it). +func (s *Service) UpdateConfig(cfg config.InboxConfig) { + s.mu.Lock() + s.cfg = cfg + s.mu.Unlock() +} + +// Settings returns the operator view of the config. +func (s *Service) Settings() Settings { + cfg := s.Config() + st := Settings{ + Sources: cfg.EffectiveSources(), + Rules: cfg.Rules, + SpekRepo: cfg.Spek.Repo, + DibsURL: cfg.Dibs.URL, + } + if st.Rules == nil { + st.Rules = []config.InboxRule{} + } + if s.spekReady != nil { + st.SpekReady = s.spekReady() + } + if s.dibsTokenSet != nil { + st.DibsTokenSet = s.dibsTokenSet() + } + return st +} + +// Items lists every item across the configured sources with state applied, +// newest first. +func (s *Service) Items(ctx context.Context) ([]Item, error) { + cfg := s.Config() + var out []Item + var errs []error + for _, typ := range cfg.EffectiveSources() { + src := s.source(typ) + if src == nil { + continue + } + items, err := src.List(ctx) + if err != nil { + errs = append(errs, fmt.Errorf("%s: %w", typ, err)) + } + out = append(out, items...) + } + s.Store.Apply(out) + sortItems(out) + return out, errors.Join(errs...) +} + +func (s *Service) source(typ string) Source { + s.mu.RLock() + defer s.mu.RUnlock() + return s.sources[typ] +} + +// Find returns one item by key. +func (s *Service) Find(ctx context.Context, source, id string) (Item, bool, error) { + items, err := s.Items(ctx) + for _, it := range items { + if it.Source == source && it.ID == id { + return it, true, err + } + } + return Item{}, false, err +} + +// Unprocessed counts items with no filed route and not dismissed. +func Unprocessed(items []Item) int { + n := 0 + for _, it := range items { + if !it.Processed { + n++ + } + } + return n +} + +// Dismiss marks an item knowledge-only. +func (s *Service) Dismiss(source, id string, dismissed bool) error { + return s.Store.Dismiss(Key(source, id), dismissed) +} + +// ErrUnknownDestination is returned for a destination the service lacks. +var ErrUnknownDestination = errors.New("unknown destination") + +// Route queues it for dest and runs the delivery in the background. The +// returned state is the queued state; callers poll Items for progress. +func (s *Service) Route(ctx context.Context, it Item, req Request) (RouteState, error) { + if req.Destination == DestDismiss { + if err := s.Dismiss(it.Source, it.ID, true); err != nil { + return RouteState{}, err + } + return RouteState{Status: StatusFiled, UpdatedAt: s.now()}, nil + } + s.mu.RLock() + dest := s.dests[req.Destination] + s.mu.RUnlock() + if dest == nil { + return RouteState{}, fmt.Errorf("%w: %q", ErrUnknownDestination, req.Destination) + } + if req.Destination == DestSpekRFC && strings.TrimSpace(req.Repo) == "" { + req.Repo = s.Config().Spek.Repo + } + st, err := s.Store.Begin(it.Key(), req.Destination, req.Repo) + if err != nil { + return st, err + } + s.wg.Add(1) + run := s.Run + if run == nil { + run = func(fn func()) { go fn() } + } + run(func() { + defer s.wg.Done() + s.deliver(it, dest, req) + }) + return st, nil +} + +func (s *Service) deliver(it Item, dest Destination, req Request) { + key := it.Key() + ctx, cancel := context.WithTimeout(context.Background(), routeTimeout) + defer cancel() + if _, err := s.Store.Set(key, req.Destination, StatusRunning, Result{}, ""); err != nil { + s.Logger.Warn("inbox: persist running state", "item", key, "error", err) + } + src := s.source(it.Source) + if src == nil { + s.fail(key, req.Destination, fmt.Errorf("source %q is not configured", it.Source)) + return + } + content, err := src.Fetch(ctx, it) + if err != nil { + s.fail(key, req.Destination, fmt.Errorf("fetch: %w", err)) + return + } + res, err := dest.Deliver(ctx, it, content, req) + if err != nil { + s.fail(key, req.Destination, err) + return + } + status := StatusFiled + if res.URL == "" && res.Note != "" { + status = StatusDelegated + } + if _, err := s.Store.Set(key, req.Destination, status, res, res.Note); err != nil { + s.Logger.Warn("inbox: persist filed state", "item", key, "error", err) + } + s.Logger.Info("inbox: routed", "item", key, "destination", req.Destination, "status", status, "url", res.URL) +} + +func (s *Service) fail(key, dest string, err error) { + s.Logger.Warn("inbox: route failed", "item", key, "destination", dest, "error", err) + if _, perr := s.Store.Set(key, dest, StatusFailed, Result{}, err.Error()); perr != nil { + s.Logger.Warn("inbox: persist failed state", "item", key, "error", perr) + } +} + +// Sweep applies auto-routing rules to items the store has never seen and +// marks every listed item seen, so a rule added later never back-fires on +// old recordings. It is called after each connector sync. +func (s *Service) Sweep(ctx context.Context) (routed int, err error) { + items, lerr := s.Items(ctx) + rules := s.Config().Rules + for _, it := range items { + if s.Store.Seen(it.Key()) { + continue + } + rule, ok := FirstRule(rules, it) + if !ok { + _ = s.Store.MarkSeen(it.Key()) + continue + } + if _, rerr := s.Route(ctx, it, Request{Destination: rule.Destination, Repo: rule.Repo}); rerr != nil { + s.Logger.Warn("inbox: auto-route", "item", it.Key(), "error", rerr) + _ = s.Store.MarkSeen(it.Key()) + continue + } + routed++ + } + return routed, lerr +} + +// Wait blocks until background deliveries finish (tests, shutdown). +func (s *Service) Wait() { s.wg.Wait() } + +func (s *Service) now() time.Time { + if s.Now != nil { + return s.Now().UTC() + } + return time.Now().UTC() +} + +func sortItems(items []Item) { + for i := 1; i < len(items); i++ { + for j := i; j > 0 && items[j].RecordedAt.After(items[j-1].RecordedAt); j-- { + items[j], items[j-1] = items[j-1], items[j] + } + } +} diff --git a/src/pkg/inbox/spek.go b/src/pkg/inbox/spek.go new file mode 100644 index 0000000000..4a0e39df73 --- /dev/null +++ b/src/pkg/inbox/spek.go @@ -0,0 +1,257 @@ +package inbox + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "os" + "os/exec" + "regexp" + "strings" + "text/template" + "time" +) + +// StatusDelegated means the route was handed to a hive agent (no binary in +// the image); the agent files the RFC and the operator links it manually. +const StatusDelegated = "delegated" + +// spekTimeout bounds one `spektacular voice --publish` run. +const spekTimeout = 10 * time.Minute + +// Kicker sends a prompt to a named hive agent (the dashboard's kick path). +type Kicker func(agent, prompt string) error + +// SpekDestination files an item as a spek RFC + child issues by running +// `spektacular voice --from stdin --publish` inside the hive. When the +// binary is missing and Agent is set, it kicks that agent with a prompt +// instead (fallback until the hive image ships spektacular). +type SpekDestination struct { + Binary string + DefaultRepo string + // Token mints a GitHub token for repo; empty/nil inherits the process env. + Token func(ctx context.Context, repo string) (string, error) + // Exec runs the command; nil uses os/exec. Tests inject it. + Exec func(ctx context.Context, binary string, args []string, env []string, stdin string) (string, error) + // LookPath reports whether binary is runnable; nil uses exec.LookPath. + LookPath func(binary string) error + Agent string + Kick Kicker + Prefixes []string +} + +// Name implements Destination. +func (d *SpekDestination) Name() string { return DestSpekRFC } + +// ErrNoRepo is returned when neither the request nor config names a repo. +var ErrNoRepo = errors.New("spek_rfc: no repo given and inbox.spek.repo is unset") + +// Deliver implements Destination. +func (d *SpekDestination) Deliver(ctx context.Context, it Item, c Content, req Request) (Result, error) { + repo := strings.TrimSpace(req.Repo) + if repo == "" { + repo = strings.TrimSpace(d.DefaultRepo) + } + if repo == "" { + return Result{}, ErrNoRepo + } + title := StripPrefix(firstNonEmpty(c.Title, it.Title), d.prefixes()) + text := c.Text() + if strings.TrimSpace(text) == "" { + return Result{}, fmt.Errorf("spek_rfc: item %s has no transcript or summary", it.Key()) + } + binary := firstNonEmpty(d.Binary, "spektacular") + if err := d.lookPath(binary); err != nil { + if d.Agent != "" && d.Kick != nil { + prompt := RenderAgentPrompt(it, c, repo, title) + if kerr := d.Kick(d.Agent, prompt); kerr != nil { + return Result{}, fmt.Errorf("spek_rfc: %s unavailable and kick to %s failed: %w", binary, d.Agent, kerr) + } + return Result{Note: "delegated to agent " + d.Agent}, nil + } + return Result{}, fmt.Errorf("spek_rfc: %s is not installed in this hive (set inbox.spek.binary or inbox.spek.agent): %w", binary, err) + } + stdin := text + "\n\nProvenance: " + c.Provenance + "\n" + env := os.Environ() + if d.Token != nil { + tok, err := d.Token(ctx, repo) + if err != nil { + return Result{}, fmt.Errorf("spek_rfc: github token: %w", err) + } + if tok != "" { + env = append(env, "GITHUB_TOKEN="+tok, "GH_TOKEN="+tok) + } + } + rctx, cancel := context.WithTimeout(ctx, spekTimeout) + defer cancel() + args := []string{"voice", "--from", "stdin", "--title", title, "--publish", "--repo", repo, "--format", "json"} + out, err := d.exec(rctx, binary, args, env, stdin) + if err != nil && isUnknownFlag(out, err) { + out, err = d.exec(rctx, binary, args[:len(args)-2], env, stdin) + } + if err != nil { + return Result{}, fmt.Errorf("spek_rfc: %w: %s", err, lastLines(out, 5)) + } + res, perr := ParsePublishOutput(out) + if perr != nil { + return Result{}, fmt.Errorf("spek_rfc: %w", perr) + } + return res, nil +} + +func (d *SpekDestination) prefixes() []string { + if len(d.Prefixes) > 0 { + return d.Prefixes + } + return DefaultTitlePrefixes +} + +func (d *SpekDestination) lookPath(binary string) error { + if d.LookPath != nil { + return d.LookPath(binary) + } + _, err := exec.LookPath(binary) + return err +} + +func (d *SpekDestination) exec(ctx context.Context, binary string, args, env []string, stdin string) (string, error) { + if d.Exec != nil { + return d.Exec(ctx, binary, args, env, stdin) + } + cmd := exec.CommandContext(ctx, binary, args...) + cmd.Env = env + cmd.Stdin = strings.NewReader(stdin) + var buf bytes.Buffer + cmd.Stdout = &buf + cmd.Stderr = &buf + err := cmd.Run() + return buf.String(), err +} + +func isUnknownFlag(out string, err error) bool { + s := strings.ToLower(out + " " + err.Error()) + return strings.Contains(s, "unknown flag") || strings.Contains(s, "flag provided but not defined") || strings.Contains(s, "unknown shorthand flag") +} + +// publishJSON is the `spektacular publish --format json` envelope (see +// spektacular internal/publish.Result). +type publishJSON struct { + RFCURL string `json:"rfc_url"` + Children []struct { + URL string `json:"url"` + } `json:"children"` +} + +var issueURLRe = regexp.MustCompile(`https://[^\s"'<>)]+/[^\s"'<>)/]+/[^\s"'<>)/]+/(?:issues|pull)/\d+`) + +// ParsePublishOutput accepts either the JSON envelope (`rfc_url`, +// `children[].url`) or plain text where the first issue URL is the RFC and +// the rest are children (what `spektacular voice` prints). +func ParsePublishOutput(out string) (Result, error) { + trimmed := strings.TrimSpace(out) + if start := strings.Index(trimmed, "{"); start >= 0 { + var pj publishJSON + if dec := json.NewDecoder(strings.NewReader(trimmed[start:])); dec.Decode(&pj) == nil && pj.RFCURL != "" { + res := Result{URL: pj.RFCURL} + for _, ch := range pj.Children { + if ch.URL != "" { + res.Children = append(res.Children, ch.URL) + } + } + return res, nil + } + } + urls := issueURLRe.FindAllString(out, -1) + seen := map[string]bool{} + var uniq []string + for _, u := range urls { + if !seen[u] { + seen[u] = true + uniq = append(uniq, u) + } + } + if len(uniq) == 0 { + return Result{}, fmt.Errorf("no RFC URL in spektacular output: %s", lastLines(out, 3)) + } + return Result{URL: uniq[0], Children: uniq[1:]}, nil +} + +var agentPromptTmpl = template.Must(template.New("spek").Parse(`Turn the voice recording below into a spek RFC with child issues on {{.Repo}}. + +Steps: +1. Ensure the spektacular CLI is available ("go install github.com/hivecommons/spektacular@latest" or a release binary). +2. In the {{.Repo}} clone, run "spektacular init" if .spektacular/ is missing. +3. Create a spec named "{{.Name}}" seeded from the transcript and summary below. Fill it non-interactively using your judgment; list anything unclear under "Open questions". +4. Run: spektacular publish {{.Name}} --repo {{.Repo}} --allow-draft --format json +5. Reply with the rfc_url and the child issue URLs from the JSON output. + +Provenance: {{.Provenance}} +Title: {{.Title}} + +{{if .Summary}}## Summary +{{.Summary}} + +{{end}}## Transcript +{{.Transcript}} +`)) + +// RenderAgentPrompt is the fallback kick prompt used when the hive image +// lacks the spektacular binary. +func RenderAgentPrompt(it Item, c Content, repo, title string) string { + transcript := c.Transcript + if transcript == "" { + transcript = c.Markdown + } + var buf bytes.Buffer + _ = agentPromptTmpl.Execute(&buf, map[string]string{ + "Repo": repo, + "Name": specName(title), + "Title": title, + "Summary": c.Summary, + "Transcript": transcript, + "Provenance": c.Provenance, + }) + return buf.String() +} + +// specName derives a kebab-case spec name from a title. +func specName(title string) string { + var b strings.Builder + dash := false + for _, r := range strings.ToLower(title) { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + b.WriteRune(r) + dash = false + } else if !dash && b.Len() > 0 { + b.WriteByte('-') + dash = true + } + } + name := strings.TrimRight(b.String(), "-") + if name == "" { + name = "voice-rfc" + } + if len(name) > 60 { + name = strings.TrimRight(name[:60], "-") + } + return name +} + +func firstNonEmpty(vals ...string) string { + for _, v := range vals { + if strings.TrimSpace(v) != "" { + return strings.TrimSpace(v) + } + } + return "" +} + +func lastLines(s string, n int) string { + lines := strings.Split(strings.TrimSpace(s), "\n") + if len(lines) > n { + lines = lines[len(lines)-n:] + } + return strings.TrimSpace(strings.Join(lines, "\n")) +} diff --git a/src/pkg/inbox/store.go b/src/pkg/inbox/store.go new file mode 100644 index 0000000000..17f3e0caec --- /dev/null +++ b/src/pkg/inbox/store.go @@ -0,0 +1,218 @@ +package inbox + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "sort" + "sync" + "time" +) + +// entryState is the persisted per-item state, keyed by Key(source, id). +type entryState struct { + Routes map[string]*RouteState `json:"routes,omitempty"` + Dismissed bool `json:"dismissed,omitempty"` +} + +// Store persists route states as one JSON file next to connector state. +type Store struct { + Path string + Now func() time.Time + + mu sync.Mutex + entries map[string]*entryState + loaded bool +} + +// NewStore returns a store backed by path (created on first save). +func NewStore(path string) *Store { + return &Store{Path: path} +} + +func (s *Store) now() time.Time { + if s.Now != nil { + return s.Now().UTC() + } + return time.Now().UTC() +} + +func (s *Store) load() { + if s.loaded { + return + } + s.loaded = true + s.entries = map[string]*entryState{} + if s.Path == "" { + return + } + data, err := os.ReadFile(s.Path) + if err != nil { + return + } + _ = json.Unmarshal(data, &s.entries) + if s.entries == nil { + s.entries = map[string]*entryState{} + } +} + +func (s *Store) save() error { + if s.Path == "" { + return nil + } + data, err := json.MarshalIndent(s.entries, "", " ") + if err != nil { + return err + } + if err := os.MkdirAll(filepath.Dir(s.Path), 0o755); err != nil { + return err + } + tmp := s.Path + ".tmp" + if err := os.WriteFile(tmp, data, 0o600); err != nil { + return err + } + return os.Rename(tmp, s.Path) +} + +func (s *Store) entry(key string) *entryState { + e := s.entries[key] + if e == nil { + e = &entryState{Routes: map[string]*RouteState{}} + s.entries[key] = e + } + if e.Routes == nil { + e.Routes = map[string]*RouteState{} + } + return e +} + +// Apply copies persisted state onto items and derives Processed. +func (s *Store) Apply(items []Item) { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + for i := range items { + e := s.entries[items[i].Key()] + if e == nil { + items[i].Processed = false + continue + } + items[i].Dismissed = e.Dismissed + items[i].Routes = cloneRoutes(e.Routes) + items[i].Processed = e.Dismissed + for _, r := range e.Routes { + if r.Status == StatusFiled { + items[i].Processed = true + } + } + } +} + +func cloneRoutes(in map[string]*RouteState) map[string]*RouteState { + if len(in) == 0 { + return nil + } + out := make(map[string]*RouteState, len(in)) + for k, v := range in { + c := *v + c.Children = append([]string(nil), v.Children...) + out[k] = &c + } + return out +} + +// Route returns the current state for one destination, if any. +func (s *Store) Route(key, dest string) (RouteState, bool) { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + e := s.entries[key] + if e == nil || e.Routes[dest] == nil { + return RouteState{}, false + } + return *e.Routes[dest], true +} + +// ErrRouteBusy is returned when a route is queued or running already. +var ErrRouteBusy = fmt.Errorf("route already in progress") + +// Begin transitions key/dest to queued. A queued or running route is +// rejected; filed and failed routes may be retried (filed = re-file). +func (s *Store) Begin(key, dest, repo string) (RouteState, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + e := s.entry(key) + if cur := e.Routes[dest]; cur != nil && (cur.Status == StatusQueued || cur.Status == StatusRunning) { + return *cur, ErrRouteBusy + } + st := &RouteState{Status: StatusQueued, Repo: repo, UpdatedAt: s.now()} + e.Routes[dest] = st + return *st, s.save() +} + +// Set replaces the status (and optional outcome) of key/dest. +func (s *Store) Set(key, dest, status string, res Result, errText string) (RouteState, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + e := s.entry(key) + st := e.Routes[dest] + if st == nil { + st = &RouteState{} + e.Routes[dest] = st + } + st.Status = status + st.UpdatedAt = s.now() + st.Error = errText + if status == StatusFiled { + st.URL = res.URL + st.Children = append([]string(nil), res.Children...) + } + return *st, s.save() +} + +// Dismiss marks the item knowledge-only (or clears that mark). +func (s *Store) Dismiss(key string, dismissed bool) error { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + s.entry(key).Dismissed = dismissed + return s.save() +} + +// Known returns every key the store has seen, sorted (for tests/debugging). +func (s *Store) Known() []string { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + out := make([]string, 0, len(s.entries)) + for k := range s.entries { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// Seen reports whether the store has any record of key; Sweep uses it so +// auto-rules only fire on items that arrived after the rule existed. +func (s *Store) Seen(key string) bool { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + _, ok := s.entries[key] + return ok +} + +// MarkSeen records key without any route so a later Sweep skips it. +func (s *Store) MarkSeen(key string) error { + s.mu.Lock() + defer s.mu.Unlock() + s.load() + if _, ok := s.entries[key]; ok { + return nil + } + s.entry(key) + return s.save() +} diff --git a/src/pkg/knowledge/connector/connector.go b/src/pkg/knowledge/connector/connector.go index f66542b913..953b89907a 100644 --- a/src/pkg/knowledge/connector/connector.go +++ b/src/pkg/knowledge/connector/connector.go @@ -216,7 +216,7 @@ func NewRegistry() *Registry { // DefaultRegistry returns a registry with the built-in connector types // (`git`, `document`, `github-wiki`, `repo-wiki`, `confluence`, `notion`, -// `sharepoint`, `google-drive` and `context7`) registered. +// `sharepoint`, `google-drive`, `context7`, `plaud` and `wisprflow`) registered. func DefaultRegistry() *Registry { r := NewRegistry() _ = r.Register(TypeGit, newGitConnector) @@ -228,6 +228,8 @@ func DefaultRegistry() *Registry { _ = r.Register(TypeSharePoint, newSharePointConnector) _ = r.Register(TypeRepoWiki, newRepoWikiConnector) _ = r.Register(TypeContext7, newContext7Connector) + _ = r.Register(TypePlaud, newPlaudConnector) + _ = r.Register(TypeWisprFlow, newWisprConnector) return r } diff --git a/src/pkg/knowledge/connector/connector_test.go b/src/pkg/knowledge/connector/connector_test.go index 076c37a641..4e0c8714fa 100644 --- a/src/pkg/knowledge/connector/connector_test.go +++ b/src/pkg/knowledge/connector/connector_test.go @@ -95,7 +95,7 @@ func TestRegistryRegister(t *testing.T) { if got := r.Types(); !reflect.DeepEqual(got, []string{"another", "fake"}) { t.Fatalf("Types() = %v", got) } - if got := DefaultRegistry().Types(); !reflect.DeepEqual(got, []string{TypeConfluence, TypeContext7, TypeDocument, TypeGit, TypeGitHubWiki, TypeGoogleDrive, TypeNotion, TypeRepoWiki, TypeSharePoint}) { + if got := DefaultRegistry().Types(); !reflect.DeepEqual(got, []string{TypeConfluence, TypeContext7, TypeDocument, TypeGit, TypeGitHubWiki, TypeGoogleDrive, TypeNotion, TypePlaud, TypeRepoWiki, TypeSharePoint, TypeWisprFlow}) { t.Fatalf("DefaultRegistry().Types() = %v", got) } } diff --git a/src/pkg/knowledge/connector/mcpstdio.go b/src/pkg/knowledge/connector/mcpstdio.go new file mode 100644 index 0000000000..5d916fcc72 --- /dev/null +++ b/src/pkg/knowledge/connector/mcpstdio.go @@ -0,0 +1,432 @@ +package connector + +import ( + "bufio" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "sync" + "sync/atomic" +) + +// mcpProtocolVersion is the MCP revision hive speaks as a stdio client. +const mcpProtocolVersion = "2024-11-05" + +// mcpMaxLineBytes caps one JSON-RPC line read from an MCP server. +const mcpMaxLineBytes = 32 << 20 + +// mcpClient is one connected MCP session. Call invokes a tool and returns +// the concatenated text content of the result. +type mcpClient interface { + Call(ctx context.Context, tool string, args map[string]any) (string, error) + // Notices returns side-channel text seen so far (logging notifications + // and stderr), used to surface OAuth URLs printed by the server. + Notices() string + Close() error +} + +// mcpDialer opens a fresh MCP session. +type mcpDialer func(ctx context.Context) (mcpClient, error) + +// mcpSession speaks newline-delimited JSON-RPC 2.0 over a reader/writer +// pair. Requests are serialized; notifications from the server are kept as +// notices. Close terminates the underlying process when there is one. +type mcpSession struct { + w io.Writer + lines chan []byte + done chan struct{} + nextID atomic.Int64 + mu sync.Mutex + notice strings.Builder + stderr *lockedBuffer + closer func() error + reqMu sync.Mutex +} + +type rpcMessage struct { + ID json.RawMessage `json:"id,omitempty"` + Method string `json:"method,omitempty"` + Params json.RawMessage `json:"params,omitempty"` + Result json.RawMessage `json:"result,omitempty"` + Error *struct { + Code int `json:"code"` + Message string `json:"message"` + } `json:"error,omitempty"` +} + +// newMCPSession wires a session over r/w and performs the MCP handshake. +func newMCPSession(ctx context.Context, r io.Reader, w io.Writer, stderr *lockedBuffer, closer func() error) (*mcpSession, error) { + s := &mcpSession{w: w, lines: make(chan []byte, 64), done: make(chan struct{}), stderr: stderr, closer: closer} + go s.readLoop(r) + init := map[string]any{ + "protocolVersion": mcpProtocolVersion, + "capabilities": map[string]any{}, + "clientInfo": map[string]any{"name": "hive-knowledge-connector", "version": "1"}, + } + if _, err := s.request(ctx, "initialize", init); err != nil { + _ = s.Close() + return nil, err + } + _ = s.notify("notifications/initialized", nil) + return s, nil +} + +func (s *mcpSession) readLoop(r io.Reader) { + defer close(s.done) + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 64<<10), mcpMaxLineBytes) + for sc.Scan() { + line := strings.TrimSpace(sc.Text()) + if line == "" { + continue + } + s.lines <- []byte(line) + } +} + +func (s *mcpSession) send(v any) error { + data, err := json.Marshal(v) + if err != nil { + return err + } + s.mu.Lock() + defer s.mu.Unlock() + _, err = s.w.Write(append(data, '\n')) + return err +} + +func (s *mcpSession) notify(method string, params any) error { + msg := map[string]any{"jsonrpc": "2.0", "method": method} + if params != nil { + msg["params"] = params + } + return s.send(msg) +} + +// request sends one request and waits for its reply, recording any +// notifications that arrive in between. +func (s *mcpSession) request(ctx context.Context, method string, params any) (json.RawMessage, error) { + s.reqMu.Lock() + defer s.reqMu.Unlock() + id := s.nextID.Add(1) + if err := s.send(map[string]any{"jsonrpc": "2.0", "id": id, "method": method, "params": params}); err != nil { + return nil, fmt.Errorf("mcp: sending %s: %w", method, err) + } + want := fmt.Sprintf("%d", id) + for { + select { + case <-ctx.Done(): + return nil, s.withStderr(fmt.Errorf("mcp: %s: %w", method, ctx.Err())) + case <-s.done: + // Drain replies that arrived just before the server hung up. + for { + select { + case line := <-s.lines: + if res, ok, err := s.match(line, want, method); ok { + return res, err + } + continue + default: + } + return nil, s.withStderr(fmt.Errorf("mcp: %s: server closed the connection", method)) + } + case line := <-s.lines: + if res, ok, err := s.match(line, want, method); ok { + return res, err + } + } + } +} + +// match decodes one line; ok is true when it is the reply to request id want. +func (s *mcpSession) match(line []byte, want, method string) (json.RawMessage, bool, error) { + var msg rpcMessage + if err := json.Unmarshal(line, &msg); err != nil { + return nil, false, nil + } + if len(msg.ID) == 0 || msg.Method != "" { + s.recordNotification(msg) + return nil, false, nil + } + if strings.Trim(string(msg.ID), `"`) != want { + return nil, false, nil + } + if msg.Error != nil { + return nil, true, fmt.Errorf("mcp: %s: %s (code %d)", method, msg.Error.Message, msg.Error.Code) + } + return msg.Result, true, nil +} + +func (s *mcpSession) recordNotification(msg rpcMessage) { + if len(msg.Params) == 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.notice.Len() < 64<<10 { + s.notice.Write(msg.Params) + s.notice.WriteByte('\n') + } +} + +func (s *mcpSession) withStderr(err error) error { + if s.stderr == nil { + return err + } + if tail := strings.TrimSpace(s.stderr.Tail(2000)); tail != "" { + return fmt.Errorf("%w (stderr: %s)", err, tail) + } + return err +} + +// Notices implements mcpClient. +func (s *mcpSession) Notices() string { + s.mu.Lock() + out := s.notice.String() + s.mu.Unlock() + if s.stderr != nil { + out += s.stderr.Tail(16 << 10) + } + return out +} + +// Call implements mcpClient: tools/call with the text content joined. +func (s *mcpSession) Call(ctx context.Context, tool string, args map[string]any) (string, error) { + if args == nil { + args = map[string]any{} + } + raw, err := s.request(ctx, "tools/call", map[string]any{"name": tool, "arguments": args}) + if err != nil { + return "", err + } + var res struct { + Content []struct { + Type string `json:"type"` + Text string `json:"text"` + } `json:"content"` + IsError bool `json:"isError"` + } + if err := json.Unmarshal(raw, &res); err != nil { + return "", fmt.Errorf("mcp: %s: decoding result: %w", tool, err) + } + var parts []string + for _, c := range res.Content { + if c.Type == "text" && c.Text != "" { + parts = append(parts, c.Text) + } + } + text := strings.Join(parts, "\n") + if res.IsError { + if text == "" { + text = "tool reported an error" + } + return text, fmt.Errorf("mcp: %s: %s", tool, firstLine(text)) + } + return text, nil +} + +// Close implements mcpClient. +func (s *mcpSession) Close() error { + if s.closer != nil { + return s.closer() + } + return nil +} + +// lockedBuffer is a goroutine-safe byte buffer used for child stderr. +type lockedBuffer struct { + mu sync.Mutex + buf strings.Builder +} + +func (b *lockedBuffer) Write(p []byte) (int, error) { + b.mu.Lock() + defer b.mu.Unlock() + if b.buf.Len() > 256<<10 { + return len(p), nil + } + return b.buf.Write(p) +} + +// Tail returns the last n bytes written. +func (b *lockedBuffer) Tail(n int) string { + b.mu.Lock() + defer b.mu.Unlock() + s := b.buf.String() + if len(s) > n { + return s[len(s)-n:] + } + return s +} + +// mcpChildEnvNames are the exact environment variables a stdio MCP server +// inherits from the hive process; mcpChildEnvPrefixes extend the set by +// prefix. The server is third-party code fetched at run time (the Plaud +// default is `npx -y @plaud-ai/mcp@latest`), so it gets what a Node process +// needs to start and reach the registry through the egress proxy — never the +// hub-injected credentials the hive process carries (DASHBOARD_AUTH_TOKEN, +// HIVE_TERMINAL_KEY, HIVE_UPSTREAM_PROXY_URL, provider API keys, …), which +// crux/agent strips from agent panes for the same reason. +var mcpChildEnvNames = map[string]bool{ + "PATH": true, "HOME": true, "USER": true, "LOGNAME": true, "SHELL": true, + "TMPDIR": true, "TMP": true, "TEMP": true, "TZ": true, "TERM": true, "LANG": true, "LANGUAGE": true, + "HTTP_PROXY": true, "HTTPS_PROXY": true, "NO_PROXY": true, + "http_proxy": true, "https_proxy": true, "no_proxy": true, + "SSL_CERT_FILE": true, "SSL_CERT_DIR": true, "NODE_EXTRA_CA_CERTS": true, +} + +var mcpChildEnvPrefixes = []string{"LC_", "XDG_", "NODE_", "NPM_CONFIG_", "npm_config_", "PLAUD_"} + +// mcpChildEnv filters base (KEY=VALUE pairs) down to the allowlisted names. +// A name that looks like a credential is dropped even when it matches, so a +// future prefix cannot readmit one. +func mcpChildEnv(base []string) []string { + out := make([]string, 0, len(base)) + for _, entry := range base { + key, _, ok := strings.Cut(entry, "=") + if !ok || key == "" || mcpCredentialName(key) { + continue + } + if mcpChildEnvNames[key] { + out = append(out, entry) + continue + } + for _, p := range mcpChildEnvPrefixes { + if strings.HasPrefix(key, p) { + out = append(out, entry) + break + } + } + } + return out +} + +func mcpCredentialName(key string) bool { + u := strings.ToUpper(key) + for _, marker := range []string{"TOKEN", "SECRET", "PASSWORD", "CREDENTIAL"} { + if strings.Contains(u, marker) { + return true + } + } + return false +} + +// stdioMCPDialer starts command as a child process per session. home, when +// non-empty, overrides HOME so the server keeps its own token store in a +// persistent directory. The child's environment is the allowlisted subset +// mcpChildEnv keeps, not the hive process environment. +func stdioMCPDialer(command, home string) mcpDialer { + return func(ctx context.Context) (mcpClient, error) { + argv, err := splitCommandLine(command) + if err != nil { + return nil, err + } + if len(argv) == 0 { + return nil, errors.New("mcp: command is empty") + } + cmd := exec.CommandContext(ctx, argv[0], argv[1:]...) + cmd.Env = mcpChildEnv(os.Environ()) + if home != "" { + if err := os.MkdirAll(home, 0o700); err != nil { + return nil, fmt.Errorf("mcp: creating home %s: %w", home, err) + } + cmd.Env = append(cmd.Env, "HOME="+home) + cmd.Dir = home + } + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, err + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, err + } + stderr := &lockedBuffer{} + cmd.Stderr = stderr + if err := cmd.Start(); err != nil { + return nil, fmt.Errorf("mcp: starting %q: %w", argv[0], err) + } + closer := sync.OnceValue(func() error { + _ = stdin.Close() + if cmd.Process != nil { + _ = cmd.Process.Kill() + } + return cmd.Wait() + }) + return newMCPSession(ctx, stdout, stdin, stderr, closer) + } +} + +// splitCommandLine splits a shell-like command line on whitespace, honouring +// single and double quotes. No expansion is performed. +func splitCommandLine(s string) ([]string, error) { + var out []string + var cur strings.Builder + inWord := false + var quote rune + for _, r := range s { + switch { + case quote != 0: + if r == quote { + quote = 0 + } else { + cur.WriteRune(r) + } + case r == '\'' || r == '"': + quote = r + inWord = true + case r == ' ' || r == '\t' || r == '\n': + if inWord { + out = append(out, cur.String()) + cur.Reset() + inWord = false + } + default: + cur.WriteRune(r) + inWord = true + } + } + if quote != 0 { + return nil, fmt.Errorf("mcp: unterminated quote in command %q", s) + } + if inWord { + out = append(out, cur.String()) + } + return out, nil +} + +var httpsURLRe = regexp.MustCompile(`https://[^\s"'<>\\)\]]+`) + +// firstHTTPSURL returns the first https:// URL found in s, or "". +func firstHTTPSURL(s string) string { + return httpsURLRe.FindString(s) +} + +func firstLine(s string) string { + s = strings.TrimSpace(s) + if i := strings.IndexByte(s, '\n'); i >= 0 { + s = s[:i] + } + if len(s) > 300 { + s = s[:300] + "…" + } + return s +} + +// ensureDir creates dir (and parents) and returns it. +func ensureDir(dir string) (string, error) { + if dir == "" { + return "", nil + } + if err := os.MkdirAll(filepath.Clean(dir), 0o700); err != nil { + return "", err + } + return dir, nil +} diff --git a/src/pkg/knowledge/connector/mcpstdio_test.go b/src/pkg/knowledge/connector/mcpstdio_test.go new file mode 100644 index 0000000000..2fce7e8932 --- /dev/null +++ b/src/pkg/knowledge/connector/mcpstdio_test.go @@ -0,0 +1,258 @@ +package connector + +import ( + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" +) + +// fakeMCPServer answers JSON-RPC over pipes like a stdio MCP server. tools +// maps a tool name to a handler returning (text, isError). +type fakeMCPServer struct { + tools map[string]func(args map[string]any) (string, bool) + mu sync.Mutex + calls []string + initReq map[string]any +} + +func (f *fakeMCPServer) serve(r io.Reader, w io.Writer) { + sc := bufio.NewScanner(r) + sc.Buffer(make([]byte, 64<<10), mcpMaxLineBytes) + for sc.Scan() { + var req struct { + ID json.RawMessage `json:"id"` + Method string `json:"method"` + Params struct { + Name string `json:"name"` + Arguments map[string]any `json:"arguments"` + Proto string `json:"protocolVersion"` + } `json:"params"` + } + if err := json.Unmarshal(sc.Bytes(), &req); err != nil { + continue + } + if len(req.ID) == 0 { + continue // notification + } + var result any + var rpcErr map[string]any + switch req.Method { + case "initialize": + f.mu.Lock() + f.initReq = map[string]any{"protocolVersion": req.Params.Proto} + f.mu.Unlock() + // Interleave a logging notification to prove the client skips it. + fmt.Fprintf(w, `{"jsonrpc":"2.0","method":"notifications/message","params":{"level":"info","data":"booting https://plaud.example/notice"}}`+"\n") + result = map[string]any{"protocolVersion": mcpProtocolVersion, "capabilities": map[string]any{}} + case "tools/call": + f.mu.Lock() + f.calls = append(f.calls, req.Params.Name) + h := f.tools[req.Params.Name] + f.mu.Unlock() + if h == nil { + rpcErr = map[string]any{"code": -32601, "message": "unknown tool " + req.Params.Name} + break + } + text, isErr := h(req.Params.Arguments) + result = map[string]any{"content": []map[string]any{{"type": "text", "text": text}}, "isError": isErr} + default: + rpcErr = map[string]any{"code": -32601, "message": "method not found"} + } + resp := map[string]any{"jsonrpc": "2.0", "id": json.RawMessage(req.ID)} + if rpcErr != nil { + resp["error"] = rpcErr + } else { + resp["result"] = result + } + b, _ := json.Marshal(resp) + fmt.Fprintf(w, "%s\n", b) + } +} + +func (f *fakeMCPServer) callNames() []string { + f.mu.Lock() + defer f.mu.Unlock() + return append([]string(nil), f.calls...) +} + +// pipeDialer connects an mcpSession to srv over in-memory pipes. +func pipeDialer(srv *fakeMCPServer) mcpDialer { + return func(ctx context.Context) (mcpClient, error) { + cr, sw := io.Pipe() // server → client + sr, cw := io.Pipe() // client → server + go srv.serve(sr, sw) + closer := func() error { _ = cw.Close(); _ = sw.Close(); return nil } + return newMCPSession(ctx, cr, cw, nil, closer) + } +} + +func TestMCPSessionHandshakeCallAndErrors(t *testing.T) { + srv := &fakeMCPServer{tools: map[string]func(map[string]any) (string, bool){ + "echo": func(a map[string]any) (string, bool) { return fmt.Sprint(a["x"]), false }, + "boom": func(map[string]any) (string, bool) { return "nope: not authenticated", true }, + }} + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + c, err := pipeDialer(srv)(ctx) + if err != nil { + t.Fatal(err) + } + defer c.Close() + if got, err := c.Call(ctx, "echo", map[string]any{"x": "hi"}); err != nil || got != "hi" { + t.Fatalf("echo = %q, %v", got, err) + } + if _, err := c.Call(ctx, "boom", nil); err == nil || !strings.Contains(err.Error(), "not authenticated") { + t.Fatalf("isError must surface as error, got %v", err) + } + if _, err := c.Call(ctx, "missing", nil); err == nil || !strings.Contains(err.Error(), "unknown tool") { + t.Fatalf("rpc error = %v", err) + } + if !strings.Contains(c.Notices(), "plaud.example/notice") { + t.Fatalf("notification not recorded: %q", c.Notices()) + } + if srv.initReq["protocolVersion"] != mcpProtocolVersion { + t.Fatalf("initialize params = %v", srv.initReq) + } +} + +func TestMCPSessionServerExit(t *testing.T) { + cr, sw := io.Pipe() + sr, cw := io.Pipe() + go func() { _, _ = io.Copy(io.Discard, sr) }() + go func() { + // Reply to initialize, then hang up before any tool call is answered. + fmt.Fprintln(sw, `{"jsonrpc":"2.0","id":1,"result":{}}`) + _ = sw.Close() + }() + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + s, err := newMCPSession(ctx, cr, cw, nil, nil) + if err != nil { + t.Fatal(err) + } + if _, err := s.Call(ctx, "x", nil); err == nil || !strings.Contains(err.Error(), "closed") { + t.Fatalf("want closed-connection error, got %v", err) + } +} + +func TestSplitCommandLine(t *testing.T) { + got, err := splitCommandLine(`npx -y "@plaud-ai/mcp@latest" --flag 'a b'`) + if err != nil { + t.Fatal(err) + } + want := []string{"npx", "-y", "@plaud-ai/mcp@latest", "--flag", "a b"} + if strings.Join(got, "|") != strings.Join(want, "|") { + t.Fatalf("split = %q", got) + } + if _, err := splitCommandLine(`npx "unterminated`); err == nil { + t.Fatal("unterminated quote must fail") + } +} + +func TestStdioMCPDialerSpawnsProcess(t *testing.T) { + // A tiny shell MCP server: answers initialize and one tools/call. + dir := t.TempDir() + script := filepath.Join(dir, "server.sh") + body := "read line\necho '{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{}}'\nread line\nread line\necho '{\"jsonrpc\":\"2.0\",\"id\":2,\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"pong\"}]}}'\n" + if err := os.WriteFile(script, []byte(body), 0o700); err != nil { + t.Fatal(err) + } + dial := stdioMCPDialer("sh "+script, filepath.Join(dir, "home")) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + c, err := dial(ctx) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer c.Close() + if got, err := c.Call(ctx, "ping", nil); err != nil || got != "pong" { + t.Fatalf("ping = %q, %v", got, err) + } + if _, err := os.Stat(filepath.Join(dir, "home")); err != nil { + t.Fatalf("HOME dir must be created: %v", err) + } +} + +func TestMCPChildEnvDropsHiveSecrets(t *testing.T) { + base := []string{ + "PATH=/usr/bin", + "HOME=/srv/hive", + "LC_ALL=C.UTF-8", + "NODE_OPTIONS=--max-old-space-size=512", + "npm_config_cache=/tmp/npm", + "HTTPS_PROXY=http://proxy:3128", + "NODE_EXTRA_CA_CERTS=/etc/hive/ca.pem", + "PLAUD_REGION=eu", + "DASHBOARD_AUTH_TOKEN=owner-token", + "HIVE_DASHBOARD_TOKEN=owner-token", + "HIVE_TERMINAL_KEY=terminal-key", + "HIVE_HEARTBEAT_KEY=heartbeat-key", + "HIVE_HUB_SECRET=hub-secret", + "HIVE_UPSTREAM_PROXY_URL=http://user:pass@corp:3128", + "HIVE_GITHUB_TOKEN=ghs_x", + "GITHUB_TOKEN=ghp_x", + "ANTHROPIC_API_KEY=sk-ant", + "LINEAR_CLIENT_SECRET=lin", + // Allowlisted prefix but credential-shaped: must still be dropped. + "NODE_AUTH_TOKEN=npm-token", + "PLAUD_SECRET=plaud", + "malformed-no-equals", + } + got := mcpChildEnv(base) + want := []string{ + "PATH=/usr/bin", + "HOME=/srv/hive", + "LC_ALL=C.UTF-8", + "NODE_OPTIONS=--max-old-space-size=512", + "npm_config_cache=/tmp/npm", + "HTTPS_PROXY=http://proxy:3128", + "NODE_EXTRA_CA_CERTS=/etc/hive/ca.pem", + "PLAUD_REGION=eu", + } + if strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Fatalf("mcpChildEnv =\n%s\nwant\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} + +// The spawned MCP server must not see the hub-injected credentials the hive +// process carries (hivecommons/hive#17843): the same secrets crux/agent +// strips from agent panes. +func TestStdioMCPDialerDoesNotLeakHiveEnv(t *testing.T) { + t.Setenv("DASHBOARD_AUTH_TOKEN", "dash_auth_leaked") + t.Setenv("HIVE_TERMINAL_KEY", "terminal_key_leaked") + t.Setenv("HIVE_UPSTREAM_PROXY_URL", "http://u:p@corp:3128") + t.Setenv("PLAUD_REGION", "eu") + + dir := t.TempDir() + script := filepath.Join(dir, "server.sh") + // Answers initialize, then reports the env it sees on the first tools/call. + body := "read line\necho '{\"jsonrpc\":\"2.0\",\"id\":1,\"result\":{}}'\nread line\nread line\n" + + "printf '{\"jsonrpc\":\"2.0\",\"id\":2,\"result\":{\"content\":[{\"type\":\"text\",\"text\":\"dash=%s term=%s proxy=%s region=%s path_set=%s\"}]}}\\n' " + + "\"$DASHBOARD_AUTH_TOKEN\" \"$HIVE_TERMINAL_KEY\" \"$HIVE_UPSTREAM_PROXY_URL\" \"$PLAUD_REGION\" \"${PATH:+yes}\"\n" + if err := os.WriteFile(script, []byte(body), 0o700); err != nil { + t.Fatal(err) + } + dial := stdioMCPDialer("sh "+script, filepath.Join(dir, "home")) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + c, err := dial(ctx) + if err != nil { + t.Fatalf("dial: %v", err) + } + defer c.Close() + got, err := c.Call(ctx, "env", nil) + if err != nil { + t.Fatalf("call: %v", err) + } + if got != "dash= term= proxy= region=eu path_set=yes" { + t.Fatalf("child env = %q; hub secrets must be absent, PLAUD_* and PATH present", got) + } +} diff --git a/src/pkg/knowledge/connector/plaud.go b/src/pkg/knowledge/connector/plaud.go new file mode 100644 index 0000000000..02e959cbaf --- /dev/null +++ b/src/pkg/knowledge/connector/plaud.go @@ -0,0 +1,439 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +// TypePlaud is the connector type for Plaud voice-note recordings, read +// through the official (free) Plaud MCP server over stdio. +const TypePlaud = "plaud" + +const ( + // plaudDefaultCommand launches the official Plaud MCP server. + plaudDefaultCommand = "npx -y @plaud-ai/mcp@latest" + // plaudDefaultSinceDays bounds the first sync. + plaudDefaultSinceDays = 30 + // plaudDefaultMaxFiles caps recordings fetched per sync. + plaudDefaultMaxFiles = 200 + // plaudListPageSize is the list_files page size requested. + plaudListPageSize = 50 + // plaudRelistOverlap re-lists this far behind the cursor so late + // transcripts and edited summaries are picked up. + plaudRelistOverlap = 24 * time.Hour + // plaudLoginWindow is how long a login session stays alive to receive + // the OAuth callback. + plaudLoginWindow = 10 * time.Minute + // plaudLoginReplyWait bounds how long Connect waits for the login tool + // to answer before returning whatever URL has surfaced so far. + plaudLoginReplyWait = 8 * time.Second + // plaudHomeInherit keeps the hive process HOME for the MCP server. + plaudHomeInherit = "inherit" +) + +// plaudConnector syncs Plaud recordings: list_files → get_transcript + +// get_note → one page per recording. Scope keys: command, home, since_days, +// max_files, include_audio, query, tags. No auth.env/auth.file: Plaud's +// OAuth token lives in the MCP server's own store ($HOME/.plaud). +type plaudConnector struct { + cfg ConnectorConfig + logger *slog.Logger + dial mcpDialer + now func() time.Time + stateDir string + // home is the MCP server's HOME (token store); empty inherits the + // process HOME. + home string + // callbackBase is where a pasted OAuth redirect is forwarded while a + // login session is alive (the MCP server's fixed localhost listener). + callbackBase string + + mu sync.Mutex + login *plaudLogin + auth AuthState +} + +// plaudLogin is an in-flight OAuth login kept alive for the callback. +type plaudLogin struct { + client mcpClient + cancel context.CancelFunc + result string + done chan struct{} +} + +func newPlaudConnector(cfg ConnectorConfig, deps Deps) (Connector, error) { + c := &plaudConnector{cfg: cfg, logger: deps.logger(), now: time.Now, stateDir: deps.StateDir} + home := cfg.ScopeValue("home") + switch { + case home == plaudHomeInherit: + home = "" + case home == "" && deps.StateDir != "": + home = filepath.Join(deps.StateDir, "home") + } + if _, err := ensureDir(home); err != nil { + return nil, fmt.Errorf("plaud: %w", err) + } + command := cfg.ScopeValue("command") + if command == "" { + command = plaudDefaultCommand + } + c.dial = stdioMCPDialer(command, home) + c.home = home + c.callbackBase = plaudCallbackBase + return c, nil +} + +func (c *plaudConnector) Type() string { return TypePlaud } + +// Validate checks the scope. Plaud never takes a hive-managed credential. +func (c *plaudConnector) Validate(cfg ConnectorConfig) error { + if cfg.Auth.Configured() { + return fmt.Errorf("plaud: auth.env/auth.file are not used; Plaud signs in through its own OAuth flow (Connect in the dashboard)") + } + if cmd := cfg.ScopeValue("command"); cmd != "" { + if _, err := splitCommandLine(cmd); err != nil { + return err + } + } + if h := cfg.ScopeValue("home"); h != "" && h != plaudHomeInherit && !filepath.IsAbs(h) { + return fmt.Errorf("plaud: scope.home must be an absolute path or %q", plaudHomeInherit) + } + for _, key := range []string{"since_days", "max_files"} { + if v := cfg.ScopeValue(key); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n <= 0 { + return fmt.Errorf("plaud: scope.%s must be a positive integer", key) + } + } + } + if v := cfg.ScopeValue("include_audio"); v != "" { + if _, err := strconv.ParseBool(v); err != nil { + return fmt.Errorf("plaud: scope.include_audio must be true or false") + } + } + return nil +} + +func (c *plaudConnector) scopeInt(key string, def int) int { + n, err := strconv.Atoi(c.cfg.ScopeValue(key)) + if err != nil || n <= 0 { + return def + } + return n +} + +func (c *plaudConnector) includeAudio() bool { + b, _ := strconv.ParseBool(c.cfg.ScopeValue("include_audio")) + return b +} + +// plaudCursor is the newest recording time seen, RFC3339. +func parsePlaudCursor(cur Cursor) time.Time { + t, err := time.Parse(time.RFC3339, string(cur)) + if err != nil { + return time.Time{} + } + return t +} + +// Sync lists recordings created since the cursor (minus an overlap window) +// and emits one page per recording. The listing is incremental, so unlisted +// pages are never tombstoned. +func (c *plaudConnector) Sync(ctx context.Context, cur Cursor, emit func(Page) error) (Cursor, error) { + client, err := c.dial(ctx) + if err != nil { + return cur, err + } + defer func() { _ = client.Close() }() + + since := parsePlaudCursor(cur) + if since.IsZero() { + since = c.now().Add(-time.Duration(c.scopeInt("since_days", plaudDefaultSinceDays)) * 24 * time.Hour) + } else { + since = since.Add(-plaudRelistOverlap) + } + files, err := c.listFiles(ctx, client, since, c.scopeInt("max_files", plaudDefaultMaxFiles)) + if err != nil { + return cur, err + } + newest := parsePlaudCursor(cur) + var skipped []string + for _, f := range files { + if err := ctx.Err(); err != nil { + return cur, err + } + page, err := c.fetchPage(ctx, client, f) + if err != nil { + skipped = append(skipped, fmt.Sprintf("%s: %v", f.ID, err)) + continue + } + if err := emit(page); err != nil { + return cur, err + } + if f.CreatedAt.After(newest) { + newest = f.CreatedAt + } + } + c.refreshAuth(ctx, client) + next := cur + if !newest.IsZero() { + next = Cursor(newest.UTC().Format(time.RFC3339)) + } + if len(skipped) > 0 { + return next, &PartialError{Err: fmt.Errorf("plaud: %d recording(s) skipped: %s", len(skipped), strings.Join(skipped, "; "))} + } + return next, nil +} + +// listFiles pages through list_files until a page is empty, repeats, or the +// cap is hit. Recordings older than since are dropped client-side as well, +// in case the server ignores date_from. +func (c *plaudConnector) listFiles(ctx context.Context, client mcpClient, since time.Time, maxFiles int) ([]plaudFile, error) { + seen := map[string]bool{} + var out []plaudFile + for page := 1; len(out) < maxFiles; page++ { + args := map[string]any{"page": page, "page_size": plaudListPageSize} + if !since.IsZero() { + args["date_from"] = since.UTC().Format("2006-01-02") + } + if q := c.cfg.ScopeValue("query"); q != "" { + args["query"] = q + } + text, err := client.Call(ctx, "list_files", args) + if err != nil { + return nil, plaudAuthError(err) + } + files, err := decodePlaudFiles(text) + if err != nil { + return nil, err + } + added := 0 + for _, f := range files { + if f.ID == "" || seen[f.ID] { + continue + } + seen[f.ID] = true + if !since.IsZero() && !f.CreatedAt.IsZero() && f.CreatedAt.Before(since) { + continue + } + out = append(out, f) + added++ + if len(out) >= maxFiles { + break + } + } + if added == 0 || len(files) < plaudListPageSize { + break + } + } + return out, nil +} + +// fetchPage builds the knowledge page for one recording. +func (c *plaudConnector) fetchPage(ctx context.Context, client mcpClient, f plaudFile) (Page, error) { + note, transcript, err := c.fetchNoteAndTranscript(ctx, client, f.ID) + if err != nil { + return Page{}, err + } + audioURL := "" + if c.includeAudio() { + if text, err := client.Call(ctx, "get_file", map[string]any{"file_id": f.ID}); err == nil { + audioURL = plaudPresignedURL(text) + } + } + title := strings.TrimSpace(f.Name) + if title == "" { + title = "Plaud recording " + f.ID + } + var md strings.Builder + fmt.Fprintf(&md, "# %s\n\n", title) + if !f.StartAt.IsZero() { + fmt.Fprintf(&md, "- **Recorded:** %s\n", f.StartAt.UTC().Format(time.RFC3339)) + } else if !f.CreatedAt.IsZero() { + fmt.Fprintf(&md, "- **Recorded:** %s\n", f.CreatedAt.UTC().Format(time.RFC3339)) + } + if f.DurationMS > 0 { + fmt.Fprintf(&md, "- **Duration:** %s\n", (time.Duration(f.DurationMS) * time.Millisecond).Round(time.Second)) + } + if audioURL != "" { + fmt.Fprintf(&md, "- **Audio (24h link):** %s\n", audioURL) + } + md.WriteString("\n") + if strings.TrimSpace(note) != "" { + md.WriteString("## Summary (Plaud)\n\n") + md.WriteString(strings.TrimSpace(note)) + md.WriteString("\n\n") + } + if strings.TrimSpace(transcript) != "" { + md.WriteString("## Transcript\n\n") + md.WriteString(strings.TrimSpace(transcript)) + md.WriteString("\n") + } + attrs := map[string]string{"plaud_file_id": f.ID} + if f.DurationMS > 0 { + attrs["duration_ms"] = strconv.FormatInt(f.DurationMS, 10) + } + if !f.StartAt.IsZero() { + attrs["recorded_at"] = f.StartAt.UTC().Format(time.RFC3339) + } + if tags := c.cfg.ScopeValue("tags"); tags != "" { + attrs["tags"] = tags + } + if f.Serial != "" { + attrs["device_serial"] = f.Serial + } + updated := f.CreatedAt + if f.StartAt.After(updated) { + updated = f.StartAt + } + return Page{ + ID: f.ID, + Title: title, + Markdown: md.String(), + UpdatedAt: updated, + Path: []string{"Plaud"}, + Attrs: attrs, + }, nil +} + +// plaudAuthError rewrites "not authenticated" tool errors into a hint that +// points at the dashboard Connect button. +func plaudAuthError(err error) error { + if err == nil { + return nil + } + msg := strings.ToLower(err.Error()) + if strings.Contains(msg, "401") || strings.Contains(msg, "not authenticated") || strings.Contains(msg, "unauthorized") || strings.Contains(msg, "login") { + return fmt.Errorf("plaud: not connected — use Connect in the Knowledge → Integrations → Plaud card (%w)", err) + } + return err +} + +// Attrs returned by the MCP server for one recording. +type plaudFile struct { + ID string + Name string + CreatedAt time.Time + StartAt time.Time + DurationMS int64 + Serial string +} + +// decodePlaudFiles accepts a JSON array of recordings, or an object wrapping +// one under files/items/data/results/recordings. +func decodePlaudFiles(text string) ([]plaudFile, error) { + text = strings.TrimSpace(text) + if text == "" { + return nil, nil + } + var raw any + if err := json.Unmarshal([]byte(text), &raw); err != nil { + if i := strings.IndexAny(text, "[{"); i > 0 { + return decodePlaudFiles(text[i:]) + } + return nil, fmt.Errorf("plaud: list_files returned non-JSON output: %s", firstLine(text)) + } + var items []any + switch v := raw.(type) { + case []any: + items = v + case map[string]any: + for _, key := range []string{"files", "items", "data", "results", "recordings"} { + if arr, ok := v[key].([]any); ok { + items = arr + break + } + } + if items == nil { + if _, ok := v["id"]; ok { + items = []any{v} + } + } + } + out := make([]plaudFile, 0, len(items)) + for _, it := range items { + m, ok := it.(map[string]any) + if !ok { + continue + } + f := plaudFile{ + ID: plaudString(m, "id", "file_id"), + Name: plaudString(m, "name", "title", "filename"), + Serial: plaudString(m, "serial_number"), + CreatedAt: plaudTime(m, "created_at", "create_time"), + StartAt: plaudTime(m, "start_at", "start_time"), + } + if d, ok := m["duration"].(float64); ok { + f.DurationMS = int64(d) + } + if f.ID != "" { + out = append(out, f) + } + } + return out, nil +} + +func plaudString(m map[string]any, keys ...string) string { + for _, k := range keys { + switch v := m[k].(type) { + case string: + if s := strings.TrimSpace(v); s != "" { + return s + } + case float64: + return strconv.FormatInt(int64(v), 10) + } + } + return "" +} + +func plaudTime(m map[string]any, keys ...string) time.Time { + for _, k := range keys { + switch v := m[k].(type) { + case string: + for _, layout := range []string{time.RFC3339Nano, time.RFC3339, "2006-01-02 15:04:05", "2006-01-02"} { + if t, err := time.Parse(layout, strings.TrimSpace(v)); err == nil { + return t + } + } + case float64: + // Epoch seconds or milliseconds. + if v > 1e12 { + return time.UnixMilli(int64(v)).UTC() + } + if v > 0 { + return time.Unix(int64(v), 0).UTC() + } + } + } + return time.Time{} +} + +// fetchNoteAndTranscript renders the note and transcript of one recording. +// It fails only when neither is available. +func (c *plaudConnector) fetchNoteAndTranscript(ctx context.Context, client mcpClient, id string) (note, transcript string, err error) { + noteText, noteErr := client.Call(ctx, "get_note", map[string]any{"file_id": id}) + transcriptText, trErr := client.Call(ctx, "get_transcript", map[string]any{"file_id": id}) + if noteErr != nil && trErr != nil { + return "", "", plaudAuthError(trErr) + } + if noteErr != nil { + noteText = "" + } + if trErr != nil { + transcriptText = "" + } + note = renderPlaudNote(noteText) + transcript = renderPlaudTranscript(transcriptText) + if strings.TrimSpace(note) == "" && strings.TrimSpace(transcript) == "" { + return "", "", fmt.Errorf("no note or transcript available yet") + } + return note, transcript, nil +} diff --git a/src/pkg/knowledge/connector/plaud_auth.go b/src/pkg/knowledge/connector/plaud_auth.go new file mode 100644 index 0000000000..710ea2d291 --- /dev/null +++ b/src/pkg/knowledge/connector/plaud_auth.go @@ -0,0 +1,374 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" +) + +// AuthState describes a connector's external sign-in. Connectors that own +// their credential flow (OAuth through a vendor MCP server) report it here; +// the dashboard renders Connect/Disconnect from it. Tokens never appear. +type AuthState struct { + // Connected is true when the upstream reports a signed-in account. + Connected bool `json:"connected"` + // Account is the signed-in identity (email or display name), if known. + Account string `json:"account,omitempty"` + // Pending is true while a login awaits the user's browser authorization. + Pending bool `json:"pending"` + // AuthURL is the authorization URL the user must open, while pending. + AuthURL string `json:"auth_url,omitempty"` + // Message is the last human-readable login/logout outcome. + Message string `json:"message,omitempty"` + CheckedAt time.Time `json:"checked_at,omitempty"` +} + +// Authenticator is optionally implemented by connectors whose credential is +// obtained interactively rather than from auth.env/auth.file. +type Authenticator interface { + // AuthStatus reports the current sign-in state. It may contact the + // upstream when refresh is true; otherwise it returns the cached state. + AuthStatus(ctx context.Context, refresh bool) AuthState + // Connect begins the sign-in flow and returns the state, including the + // URL the user must open when the flow needs a browser. + Connect(ctx context.Context) (AuthState, error) + // Disconnect signs out and revokes the stored credential upstream. + Disconnect(ctx context.Context) (AuthState, error) +} + +var _ Authenticator = (*plaudConnector)(nil) + +// AuthStatus implements Authenticator. +func (c *plaudConnector) AuthStatus(ctx context.Context, refresh bool) AuthState { + c.mu.Lock() + st := c.auth + login := c.login + c.mu.Unlock() + if login != nil { + select { + case <-login.done: + st.Pending = false + if login.result != "" { + st.Message = firstLine(login.result) + } + default: + st.Pending = true + if st.AuthURL == "" { + st.AuthURL = firstHTTPSURL(login.client.Notices()) + } + } + } + if !refresh || st.Pending { + return st + } + client, err := c.dial(ctx) + if err != nil { + st.Message = err.Error() + st.CheckedAt = c.now().UTC() + c.setAuth(st) + return st + } + defer func() { _ = client.Close() }() + c.refreshAuth(ctx, client) + c.mu.Lock() + defer c.mu.Unlock() + return c.auth +} + +// refreshAuth asks get_current_user and caches the answer. +func (c *plaudConnector) refreshAuth(ctx context.Context, client mcpClient) { + st := AuthState{CheckedAt: c.now().UTC()} + text, err := client.Call(ctx, "get_current_user", nil) + switch { + case err != nil: + st.Message = firstLine(err.Error()) + default: + st.Connected = true + st.Account = plaudAccountName(text) + } + c.setAuth(st) +} + +func (c *plaudConnector) setAuth(st AuthState) { + c.mu.Lock() + c.auth = st + c.mu.Unlock() +} + +// Connect implements Authenticator: it starts a long-lived MCP session, +// calls the server's login tool and keeps the process alive for +// plaudLoginWindow so the OAuth callback (localhost:8199) can land. The +// authorization URL is taken from the tool reply or the server's log output. +func (c *plaudConnector) Connect(ctx context.Context) (AuthState, error) { + c.mu.Lock() + if l := c.login; l != nil { + select { + case <-l.done: + c.login = nil + default: + c.mu.Unlock() + return c.AuthStatus(ctx, false), nil + } + } + c.mu.Unlock() + + loginCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), plaudLoginWindow) + client, err := c.dial(loginCtx) + if err != nil { + cancel() + return c.AuthStatus(ctx, false), err + } + l := &plaudLogin{client: client, cancel: cancel, done: make(chan struct{})} + c.mu.Lock() + c.login = l + c.auth = AuthState{Pending: true, CheckedAt: c.now().UTC()} + c.mu.Unlock() + + go c.runLogin(loginCtx, l) + + timer := time.NewTimer(plaudLoginReplyWait) + defer timer.Stop() + select { + case <-l.done: + case <-timer.C: + case <-ctx.Done(): + } + return c.AuthStatus(ctx, false), nil +} + +func (c *plaudConnector) runLogin(ctx context.Context, l *plaudLogin) { + defer close(l.done) + defer l.cancel() + text, err := l.client.Call(ctx, "login", nil) + if err != nil { + text = err.Error() + } + l.result = text + st := AuthState{CheckedAt: c.now().UTC(), Message: firstLine(text)} + if url := firstHTTPSURL(text); url != "" && err == nil { + // The tool answered with the URL only: the callback still has to + // arrive, so keep the session alive until the window closes. + c.mu.Lock() + c.auth.AuthURL = url + c.auth.Pending = true + c.auth.Message = "" + c.mu.Unlock() + if !c.waitForLogin(ctx, l.client) { + st.Message = "authorization not completed before the login window closed" + st.AuthURL = "" + } else { + c.refreshAuth(ctx, l.client) + _ = l.client.Close() + c.clearLogin(l) + return + } + } else if err == nil { + c.refreshAuth(ctx, l.client) + _ = l.client.Close() + c.clearLogin(l) + return + } + _ = l.client.Close() + c.mu.Lock() + c.auth = st + c.mu.Unlock() + c.clearLogin(l) +} + +// waitForLogin polls get_current_user until it succeeds or ctx expires. +func (c *plaudConnector) waitForLogin(ctx context.Context, client mcpClient) bool { + for { + if _, err := client.Call(ctx, "get_current_user", nil); err == nil { + return true + } + if sleepCtx(ctx, 3*time.Second) != nil { + return false + } + } +} + +func (c *plaudConnector) clearLogin(l *plaudLogin) { + c.mu.Lock() + if c.login == l { + c.login = nil + } + c.auth.Pending = false + c.mu.Unlock() +} + +// Disconnect implements Authenticator via the server's logout tool. +func (c *plaudConnector) Disconnect(ctx context.Context) (AuthState, error) { + c.mu.Lock() + if l := c.login; l != nil { + l.cancel() + c.login = nil + } + c.mu.Unlock() + client, err := c.dial(ctx) + if err != nil { + return c.AuthStatus(ctx, false), err + } + defer func() { _ = client.Close() }() + text, err := client.Call(ctx, "logout", nil) + st := AuthState{CheckedAt: c.now().UTC()} + if err != nil { + st.Message = firstLine(err.Error()) + c.setAuth(st) + return st, err + } + st.Message = firstLine(text) + if st.Message == "" { + st.Message = "signed out" + } + c.setAuth(st) + return st, nil +} + +// plaudAccountName extracts an identity from get_current_user output. +func plaudAccountName(text string) string { + text = strings.TrimSpace(text) + var m map[string]any + if err := json.Unmarshal([]byte(text), &m); err == nil { + for _, key := range []string{"email", "name", "nickname", "username", "id"} { + if s := plaudString(m, key); s != "" { + return s + } + } + if u, ok := m["user"].(map[string]any); ok { + return plaudAccountName(mustJSON(u)) + } + } + return firstLine(text) +} + +func mustJSON(v any) string { + b, _ := json.Marshal(v) + return string(b) +} + +// renderPlaudNote turns get_note output into markdown. JSON replies are +// flattened: note_list entries and content/summary/markdown fields. +func renderPlaudNote(text string) string { + text = strings.TrimSpace(text) + if text == "" || !strings.HasPrefix(text, "{") && !strings.HasPrefix(text, "[") { + return text + } + var raw any + if err := json.Unmarshal([]byte(text), &raw); err != nil { + return text + } + var parts []string + var walk func(v any) + walk = func(v any) { + switch t := v.(type) { + case string: + if s := strings.TrimSpace(t); s != "" { + parts = append(parts, s) + } + case []any: + for _, it := range t { + walk(it) + } + case map[string]any: + for _, key := range []string{"note_list", "notes", "markdown", "content", "summary", "text", "note"} { + if inner, ok := t[key]; ok { + walk(inner) + return + } + } + if s := plaudString(t, "title"); s != "" { + parts = append(parts, "### "+s) + } + } + } + walk(raw) + // Parsed JSON with no text means the note is not ready yet. + return strings.Join(parts, "\n\n") +} + +// renderPlaudTranscript turns get_transcript output into "[mm:ss] Speaker: +// text" lines when it is JSON; plain text passes through. +func renderPlaudTranscript(text string) string { + text = strings.TrimSpace(text) + if text == "" || !strings.HasPrefix(text, "{") && !strings.HasPrefix(text, "[") { + return text + } + var raw any + if err := json.Unmarshal([]byte(text), &raw); err != nil { + return text + } + var segs []any + switch t := raw.(type) { + case []any: + segs = t + case map[string]any: + for _, key := range []string{"source_list", "segments", "transcript", "items", "data"} { + if arr, ok := t[key].([]any); ok { + segs = arr + break + } + if s, ok := t[key].(string); ok { + return strings.TrimSpace(s) + } + } + } + var b strings.Builder + for _, s := range segs { + m, ok := s.(map[string]any) + if !ok { + if str, ok := s.(string); ok { + b.WriteString(str + "\n") + } + continue + } + content := plaudString(m, "text", "content", "sentence") + if content == "" { + continue + } + if ts := plaudTimestamp(m); ts != "" { + fmt.Fprintf(&b, "[%s] ", ts) + } + if sp := plaudString(m, "speaker", "speaker_name", "speaker_id"); sp != "" { + b.WriteString(sp + ": ") + } + b.WriteString(content + "\n") + } + // Parsed JSON with no segments means the transcript is not ready yet. + return b.String() +} + +// plaudTimestamp formats a segment start (ms or s) as mm:ss or h:mm:ss. +func plaudTimestamp(m map[string]any) string { + for _, key := range []string{"start_time", "start", "start_ms", "begin"} { + switch v := m[key].(type) { + case float64: + // Plaud reports segment offsets in milliseconds. + d := (time.Duration(v) * time.Millisecond).Round(time.Second) + h := int(d.Hours()) + mm := int(d.Minutes()) % 60 + ss := int(d.Seconds()) % 60 + if h > 0 { + return fmt.Sprintf("%d:%02d:%02d", h, mm, ss) + } + return fmt.Sprintf("%02d:%02d", mm, ss) + case string: + if s := strings.TrimSpace(v); s != "" { + return s + } + } + } + return "" +} + +// plaudPresignedURL pulls the presigned audio URL out of get_file output. +func plaudPresignedURL(text string) string { + var m map[string]any + if err := json.Unmarshal([]byte(strings.TrimSpace(text)), &m); err == nil { + if u := plaudString(m, "presigned_url", "audio_url", "url"); u != "" { + return u + } + } + return firstHTTPSURL(text) +} diff --git a/src/pkg/knowledge/connector/plaud_auth_complete.go b/src/pkg/knowledge/connector/plaud_auth_complete.go new file mode 100644 index 0000000000..c129fb93bd --- /dev/null +++ b/src/pkg/knowledge/connector/plaud_auth_complete.go @@ -0,0 +1,159 @@ +package connector + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "time" +) + +// AuthCompletion carries what the operator pasted to finish a sign-in whose +// OAuth callback cannot reach the hive: either the full redirect URL the +// browser landed on, or the vendor token file contents from a laptop login. +type AuthCompletion struct { + RedirectURL string `json:"redirect_url,omitempty"` + Tokens string `json:"tokens,omitempty"` +} + +// AuthCompleter is optionally implemented by Authenticators whose OAuth +// client only allows a fixed localhost redirect (Plaud: localhost:8199). A +// hosted hive never receives that callback, so the dashboard offers two +// completions: forward the pasted redirect URL to the in-pod listener while +// the login session is alive, or install pasted tokens directly. +type AuthCompleter interface { + CompleteAuth(ctx context.Context, c AuthCompletion) (AuthState, error) +} + +var _ AuthCompleter = (*plaudConnector)(nil) + +const ( + // plaudCallbackBase is the MCP server's fixed OAuth listener. + plaudCallbackBase = "http://127.0.0.1:8199" + // plaudCallbackPath is the path the vendor registered as redirect URI. + plaudCallbackPath = "/auth/callback" + // plaudTokenFile is the MCP server's token store relative to HOME. + plaudTokenFile = ".plaud/tokens-mcp.json" + // plaudCallbackTimeout bounds the forwarded callback request. + plaudCallbackTimeout = 20 * time.Second + // plaudCompleteSettle is how long CompleteAuth waits for the login + // goroutine to notice the new session before reporting state. + plaudCompleteSettle = 6 * time.Second +) + +// ErrNoPendingLogin is returned when a redirect URL is pasted but no login +// session is alive to receive it (Connect must be called first). +var ErrNoPendingLogin = errors.New("plaud: no sign-in is pending; press Connect first, then paste the redirect URL within the login window") + +// CompleteAuth implements AuthCompleter. +func (c *plaudConnector) CompleteAuth(ctx context.Context, comp AuthCompletion) (AuthState, error) { + switch { + case strings.TrimSpace(comp.Tokens) != "": + if err := c.installTokens(comp.Tokens); err != nil { + return c.AuthStatus(ctx, false), err + } + c.mu.Lock() + if l := c.login; l != nil { + l.cancel() + c.login = nil + } + c.auth = AuthState{Pending: false, CheckedAt: c.now().UTC(), Message: "tokens installed"} + c.mu.Unlock() + return c.AuthStatus(ctx, true), nil + case strings.TrimSpace(comp.RedirectURL) != "": + return c.forwardRedirect(ctx, comp.RedirectURL) + default: + return c.AuthStatus(ctx, false), errors.New("plaud: redirect_url or tokens is required") + } +} + +// forwardRedirect replays the pasted callback against the MCP server's +// localhost listener inside the pod. +func (c *plaudConnector) forwardRedirect(ctx context.Context, raw string) (AuthState, error) { + u, err := url.Parse(strings.TrimSpace(raw)) + if err != nil || u.RawQuery == "" { + return c.AuthStatus(ctx, false), errors.New("plaud: paste the full URL the browser landed on (it contains ?code=...&state=...)") + } + if u.Query().Get("code") == "" { + return c.AuthStatus(ctx, false), errors.New("plaud: the pasted URL has no code parameter") + } + c.mu.Lock() + l := c.login + c.mu.Unlock() + if l == nil { + return c.AuthStatus(ctx, false), ErrNoPendingLogin + } + select { + case <-l.done: + return c.AuthStatus(ctx, false), ErrNoPendingLogin + default: + } + target := strings.TrimRight(c.callbackBase, "/") + plaudCallbackPath + "?" + u.RawQuery + rctx, cancel := context.WithTimeout(ctx, plaudCallbackTimeout) + defer cancel() + req, err := http.NewRequestWithContext(rctx, http.MethodGet, target, nil) + if err != nil { + return c.AuthStatus(ctx, false), err + } + resp, err := http.DefaultClient.Do(req) + if err != nil { + return c.AuthStatus(ctx, false), fmt.Errorf("plaud: forwarding callback to the login session: %w", err) + } + body, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + _ = resp.Body.Close() + if resp.StatusCode >= 400 { + return c.AuthStatus(ctx, false), fmt.Errorf("plaud: callback rejected (%s): %s", resp.Status, firstLine(string(body))) + } + // The login goroutine polls get_current_user and flips state itself; + // give it a moment so the caller sees the outcome. + settle, cancelSettle := context.WithTimeout(ctx, plaudCompleteSettle) + defer cancelSettle() + select { + case <-l.done: + case <-settle.Done(): + } + return c.AuthStatus(ctx, false), nil +} + +// installTokens validates and writes a pasted tokens-mcp.json into the +// connector's HOME so the MCP server picks it up on next start. +func (c *plaudConnector) installTokens(raw string) error { + var tok struct { + AccessToken string `json:"access_token"` + RefreshToken string `json:"refresh_token"` + } + if err := json.Unmarshal([]byte(raw), &tok); err != nil { + return fmt.Errorf("plaud: tokens must be the JSON contents of ~/.plaud/tokens-mcp.json: %w", err) + } + if tok.AccessToken == "" && tok.RefreshToken == "" { + return errors.New("plaud: tokens JSON has neither access_token nor refresh_token") + } + home := c.home + if home == "" { + h, err := os.UserHomeDir() + if err != nil { + return fmt.Errorf("plaud: resolving HOME: %w", err) + } + home = h + } + path := filepath.Join(home, plaudTokenFile) + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("plaud: %w", err) + } + return os.WriteFile(path, []byte(strings.TrimSpace(raw)+"\n"), 0o600) +} + +// TokenPath returns where installTokens writes (tests). +func (c *plaudConnector) tokenPath() string { + home := c.home + if home == "" { + home, _ = os.UserHomeDir() + } + return filepath.Join(home, plaudTokenFile) +} diff --git a/src/pkg/knowledge/connector/plaud_auth_complete_test.go b/src/pkg/knowledge/connector/plaud_auth_complete_test.go new file mode 100644 index 0000000000..a4a86c5b1a --- /dev/null +++ b/src/pkg/knowledge/connector/plaud_auth_complete_test.go @@ -0,0 +1,79 @@ +package connector + +import ( + "context" + "net/http" + "net/http/httptest" + "os" + "strings" + "testing" +) + +func TestPlaudCompleteAuthInstallsPastedTokens(t *testing.T) { + srv := newPlaudTestServer() + pc := newTestPlaud(t, srv, nil) + raw := `{"access_token":"at","refresh_token":"rt","token_type":"Bearer","expires_at":1}` + st, err := pc.CompleteAuth(context.Background(), AuthCompletion{Tokens: raw}) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(pc.tokenPath()) + if err != nil { + t.Fatalf("token file not written: %v", err) + } + if strings.TrimSpace(string(data)) != raw { + t.Fatalf("token file = %q", data) + } + if !strings.HasSuffix(pc.tokenPath(), ".plaud/tokens-mcp.json") || !strings.HasPrefix(pc.tokenPath(), pc.stateDir) { + t.Fatalf("token path %q should live under the connector home", pc.tokenPath()) + } + if !st.Connected || st.Account != "andy@example.com" { + t.Fatalf("expected refreshed connected state, got %+v", st) + } +} + +func TestPlaudCompleteAuthRejectsBadInput(t *testing.T) { + pc := newTestPlaud(t, newPlaudTestServer(), nil) + ctx := context.Background() + if _, err := pc.CompleteAuth(ctx, AuthCompletion{}); err == nil { + t.Fatal("empty completion must fail") + } + if _, err := pc.CompleteAuth(ctx, AuthCompletion{Tokens: "not json"}); err == nil { + t.Fatal("non-JSON tokens must fail") + } + if _, err := pc.CompleteAuth(ctx, AuthCompletion{Tokens: `{"token_type":"Bearer"}`}); err == nil { + t.Fatal("tokens without access/refresh token must fail") + } + if _, err := pc.CompleteAuth(ctx, AuthCompletion{RedirectURL: "http://localhost:8199/auth/callback"}); err == nil { + t.Fatal("redirect without query must fail") + } + if _, err := pc.CompleteAuth(ctx, AuthCompletion{RedirectURL: "http://localhost:8199/auth/callback?code=x&state=y"}); err != ErrNoPendingLogin { + t.Fatalf("redirect without pending login: got %v, want ErrNoPendingLogin", err) + } +} + +func TestPlaudCompleteAuthForwardsRedirectToPendingLogin(t *testing.T) { + // A live login session: the fake callback listener plays the login + // goroutine, which finishes (closes done) once the callback lands. + l := &plaudLogin{done: make(chan struct{}), cancel: func() {}} + got := make(chan string, 1) + cb := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + got <- r.URL.Path + "?" + r.URL.RawQuery + close(l.done) + w.WriteHeader(http.StatusOK) + })) + defer cb.Close() + pc := newTestPlaud(t, newPlaudTestServer(), nil) + pc.callbackBase = cb.URL + pc.mu.Lock() + pc.login = l + pc.auth = AuthState{Pending: true} + pc.mu.Unlock() + _, err := pc.CompleteAuth(context.Background(), AuthCompletion{RedirectURL: "http://localhost:8199/auth/callback?code=abc&state=xyz"}) + if err != nil { + t.Fatal(err) + } + if fwd := <-got; fwd != "/auth/callback?code=abc&state=xyz" { + t.Fatalf("forwarded %q", fwd) + } +} diff --git a/src/pkg/knowledge/connector/plaud_recordings.go b/src/pkg/knowledge/connector/plaud_recordings.go new file mode 100644 index 0000000000..291acb2f32 --- /dev/null +++ b/src/pkg/knowledge/connector/plaud_recordings.go @@ -0,0 +1,83 @@ +package connector + +import ( + "context" + "sort" + "strings" + "time" +) + +// PlaudRecording is one recording as the Plaud MCP server lists it. ID is +// the Plaud file id, the same value written to the `plaud_file_id` fact attr. +type PlaudRecording struct { + ID string + Title string + RecordedAt time.Time + DurationMS int64 + Tags []string +} + +// PlaudContent is the rendered note and transcript of one recording. +type PlaudContent struct { + Summary string + Transcript string +} + +// PlaudReader reads recordings straight from Plaud without going through +// the facts a sync writes. The plaud connector implements it. +type PlaudReader interface { + // ListRecordings lists recent recordings, newest first. + ListRecordings(ctx context.Context) ([]PlaudRecording, error) + // ReadRecording returns the note and transcript of one recording. + ReadRecording(ctx context.Context, id string) (PlaudContent, error) +} + +var _ PlaudReader = (*plaudConnector)(nil) + +// ListRecordings lists recordings within the connector's since_days window, +// capped at max_files, newest first. +func (c *plaudConnector) ListRecordings(ctx context.Context) ([]PlaudRecording, error) { + client, err := c.dial(ctx) + if err != nil { + return nil, err + } + defer func() { _ = client.Close() }() + since := c.now().Add(-time.Duration(c.scopeInt("since_days", plaudDefaultSinceDays)) * 24 * time.Hour) + files, err := c.listFiles(ctx, client, since, c.scopeInt("max_files", plaudDefaultMaxFiles)) + if err != nil { + return nil, err + } + var tags []string + for _, t := range strings.Split(c.cfg.ScopeValue("tags"), ",") { + if t = strings.TrimSpace(t); t != "" { + tags = append(tags, t) + } + } + out := make([]PlaudRecording, 0, len(files)) + for _, f := range files { + rec := PlaudRecording{ID: f.ID, Title: strings.TrimSpace(f.Name), RecordedAt: f.StartAt.UTC(), DurationMS: f.DurationMS, Tags: tags} + if f.StartAt.IsZero() { + rec.RecordedAt = f.CreatedAt.UTC() + } + if rec.Title == "" { + rec.Title = "Plaud recording " + f.ID + } + out = append(out, rec) + } + sort.SliceStable(out, func(i, j int) bool { return out[i].RecordedAt.After(out[j].RecordedAt) }) + return out, nil +} + +// ReadRecording fetches the note and transcript of one recording. +func (c *plaudConnector) ReadRecording(ctx context.Context, id string) (PlaudContent, error) { + client, err := c.dial(ctx) + if err != nil { + return PlaudContent{}, err + } + defer func() { _ = client.Close() }() + note, transcript, err := c.fetchNoteAndTranscript(ctx, client, id) + if err != nil { + return PlaudContent{}, err + } + return PlaudContent{Summary: strings.TrimSpace(note), Transcript: strings.TrimSpace(transcript)}, nil +} diff --git a/src/pkg/knowledge/connector/plaud_test.go b/src/pkg/knowledge/connector/plaud_test.go new file mode 100644 index 0000000000..c0f645393f --- /dev/null +++ b/src/pkg/knowledge/connector/plaud_test.go @@ -0,0 +1,256 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" +) + +func plaudTestFiles() []map[string]any { + return []map[string]any{ + {"id": "rec-1", "name": "Standup ideas", "created_at": "2026-11-10T09:00:00Z", "start_at": "2026-11-10T08:58:00Z", "duration": 125000.0, "serial_number": "PN123"}, + {"id": "rec-2", "name": "Drive home", "created_at": "2026-11-11T18:00:00Z", "duration": 61000.0}, + {"id": "rec-old", "name": "Ancient", "created_at": "2020-01-01T00:00:00Z"}, + } +} + +func newPlaudTestServer() *fakeMCPServer { + srv := &fakeMCPServer{tools: map[string]func(map[string]any) (string, bool){}} + srv.tools["list_files"] = func(a map[string]any) (string, bool) { + if p, _ := a["page"].(float64); p > 1 { + return "[]", false + } + b, _ := json.Marshal(map[string]any{"files": plaudTestFiles()}) + return string(b), false + } + srv.tools["get_note"] = func(a map[string]any) (string, bool) { + if a["file_id"] == "rec-2" { + return "", true // still processing + } + b, _ := json.Marshal(map[string]any{"note_list": []string{"## Key points\n- ship plaud connector"}}) + return string(b), false + } + srv.tools["get_transcript"] = func(a map[string]any) (string, bool) { + if a["file_id"] == "rec-2" { + return `{"source_list":[]}`, false + } + return `{"source_list":[{"start_time":1500,"speaker":"Andy","text":"Hello hive"},{"start_time":65000,"speaker":"Andy","text":"Second line"}]}`, false + } + srv.tools["get_current_user"] = func(map[string]any) (string, bool) { + return `{"email":"andy@example.com"}`, false + } + return srv +} + +func newTestPlaud(t *testing.T, srv *fakeMCPServer, scope map[string]string) *plaudConnector { + t.Helper() + cfg := ConnectorConfig{Name: "plaud", Type: TypePlaud, Enabled: true, Layer: "personal", Scope: scope} + c, err := DefaultRegistry().New(cfg, Deps{StateDir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + pc := c.(*plaudConnector) + pc.dial = pipeDialer(srv) + pc.now = func() time.Time { return time.Date(2026, 11, 11, 20, 0, 0, 0, time.UTC) } + return pc +} + +func TestPlaudSyncEmitsPagesAndAdvancesCursor(t *testing.T) { + srv := newPlaudTestServer() + pc := newTestPlaud(t, srv, map[string]string{"tags": "voice, ideas"}) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + var pages []Page + next, err := pc.Sync(ctx, "", func(p Page) error { pages = append(pages, p); return nil }) + if err == nil || !IsPartial(err) || !strings.Contains(err.Error(), "rec-2") { + t.Fatalf("want partial error naming rec-2, got %v", err) + } + if len(pages) != 1 || pages[0].ID != "rec-1" { + t.Fatalf("pages = %+v", pages) + } + p := pages[0] + for _, want := range []string{"# Standup ideas", "**Recorded:** 2026-11-10T08:58:00Z", "**Duration:** 2m5s", "## Summary (Plaud)", "ship plaud connector", "## Transcript", "[00:02] Andy: Hello hive", "[01:05] Andy: Second line"} { + if !strings.Contains(p.Markdown, want) { + t.Errorf("markdown missing %q:\n%s", want, p.Markdown) + } + } + if p.Attrs["plaud_file_id"] != "rec-1" || p.Attrs["tags"] != "voice, ideas" || p.Attrs["device_serial"] != "PN123" || p.Attrs["duration_ms"] != "125000" { + t.Fatalf("attrs = %v", p.Attrs) + } + if next != Cursor("2026-11-10T09:00:00Z") { + t.Fatalf("cursor = %q", next) + } + // The sync refreshes the sign-in state for the dashboard card. + if st := pc.AuthStatus(ctx, false); !st.Connected || st.Account != "andy@example.com" { + t.Fatalf("auth after sync = %+v", st) + } + calls := strings.Join(srv.callNames(), ",") + if strings.Contains(calls, "get_file") { + t.Fatalf("audio must not be fetched by default: %s", calls) + } +} + +func TestPlaudSyncUsesCursorWindowAndAudio(t *testing.T) { + srv := newPlaudTestServer() + var dateFrom string + list := srv.tools["list_files"] + srv.tools["list_files"] = func(a map[string]any) (string, bool) { + dateFrom, _ = a["date_from"].(string) + return list(a) + } + srv.tools["get_file"] = func(map[string]any) (string, bool) { + return `{"id":"rec-1","presigned_url":"https://cdn.plaud.example/rec-1.mp3?sig=1"}`, false + } + pc := newTestPlaud(t, srv, map[string]string{"include_audio": "true"}) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + var pages []Page + next, err := pc.Sync(ctx, "2026-11-11T00:00:00Z", func(p Page) error { pages = append(pages, p); return nil }) + if err == nil || !IsPartial(err) { + t.Fatalf("err = %v", err) + } + // Cursor minus the 24h overlap → 2026-11-10; rec-1 (Nov 10) is re-listed. + if dateFrom != "2026-11-10" { + t.Fatalf("date_from = %q", dateFrom) + } + if len(pages) != 1 || !strings.Contains(pages[0].Markdown, "cdn.plaud.example/rec-1.mp3") { + t.Fatalf("pages = %+v", pages) + } + // The cursor never moves backwards. + if next != Cursor("2026-11-11T00:00:00Z") { + t.Fatalf("cursor = %q", next) + } +} + +func TestPlaudSyncNotAuthenticated(t *testing.T) { + srv := newPlaudTestServer() + srv.tools["list_files"] = func(map[string]any) (string, bool) { return "401 Not authenticated", true } + pc := newTestPlaud(t, srv, nil) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + _, err := pc.Sync(ctx, "", func(Page) error { return nil }) + if err == nil || !strings.Contains(err.Error(), "Connect") { + t.Fatalf("want connect hint, got %v", err) + } +} + +func TestPlaudValidate(t *testing.T) { + base := ConnectorConfig{Name: "plaud", Type: TypePlaud, Layer: "personal"} + reg := DefaultRegistry() + deps := Deps{StateDir: t.TempDir()} + if _, err := reg.New(base, deps); err != nil { + t.Fatalf("default config must validate: %v", err) + } + bad := []ConnectorConfig{ + {Name: "p", Type: TypePlaud, Layer: "personal", Auth: Auth{Env: "PLAUD_TOKEN"}}, + {Name: "p", Type: TypePlaud, Layer: "personal", Scope: map[string]string{"since_days": "zero"}}, + {Name: "p", Type: TypePlaud, Layer: "personal", Scope: map[string]string{"include_audio": "maybe"}}, + {Name: "p", Type: TypePlaud, Layer: "personal", Scope: map[string]string{"home": "relative/dir"}}, + {Name: "p", Type: TypePlaud, Layer: "personal", Scope: map[string]string{"command": `npx "oops`}}, + } + for i, cfg := range bad { + if _, err := reg.New(cfg, deps); err == nil { + t.Errorf("config %d must be rejected: %+v", i, cfg.Scope) + } + } +} + +func TestPlaudConnectCompletesLogin(t *testing.T) { + srv := newPlaudTestServer() + srv.tools["login"] = func(map[string]any) (string, bool) { + return "Open https://auth.plaud.example/authorize?state=abc to sign in", false + } + srv.tools["logout"] = func(map[string]any) (string, bool) { return "Signed out", false } + pc := newTestPlaud(t, srv, nil) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + st, err := pc.Connect(ctx) + if err != nil { + t.Fatal(err) + } + // get_current_user succeeds right away in the fake, so the login + // finishes inside the reply window and the state is already connected. + if !st.Connected || st.Account != "andy@example.com" || st.Pending { + t.Fatalf("state after connect = %+v", st) + } + st, err = pc.Disconnect(ctx) + if err != nil || st.Connected || st.Message != "Signed out" { + t.Fatalf("disconnect = %+v, %v", st, err) + } +} + +func TestPlaudConnectLoginError(t *testing.T) { + srv := newPlaudTestServer() + srv.tools["login"] = func(map[string]any) (string, bool) { return "browser could not be opened", true } + pc := newTestPlaud(t, srv, nil) + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Second) + defer cancel() + st, err := pc.Connect(ctx) + if err != nil { + t.Fatal(err) + } + if st.Connected || st.Pending || !strings.Contains(st.Message, "browser could not be opened") { + t.Fatalf("state = %+v", st) + } +} + +func TestPlaudRenderHelpers(t *testing.T) { + if got := renderPlaudNote("plain **markdown**"); got != "plain **markdown**" { + t.Fatalf("plain note = %q", got) + } + if got := renderPlaudNote(`{"note_list":[{"title":"Action items","content":"- do it"}]}`); got != "- do it" { + t.Fatalf("nested note = %q", got) + } + if got := renderPlaudTranscript("raw transcript"); got != "raw transcript" { + t.Fatalf("plain transcript = %q", got) + } + if got := renderPlaudTranscript(`{"transcript":"as string"}`); got != "as string" { + t.Fatalf("string transcript = %q", got) + } + if got := renderPlaudTranscript(`[{"start":3725000,"text":"late"}]`); !strings.HasPrefix(got, "[1:02:05] late") { + t.Fatalf("hour timestamp = %q", got) + } + files, err := decodePlaudFiles("Here are your files:\n" + `[{"id": 7, "name": "n", "created_at": 1762848000}]`) + if err != nil || len(files) != 1 || files[0].ID != "7" || files[0].CreatedAt.Year() != 2025 { + t.Fatalf("decode with prose prefix = %+v, %v", files, err) + } + if _, err := decodePlaudFiles("no json here"); err == nil { + t.Fatal("non-JSON listing must error") + } + if got := plaudAccountName(`{"user":{"name":"Andy"}}`); got != "Andy" { + t.Fatalf("account = %q", got) + } + if got := fmt.Sprint(plaudAuthError(fmt.Errorf("mcp: list_files: Unauthorized"))); !strings.Contains(got, "not connected") { + t.Fatalf("auth error = %q", got) + } +} + +func TestPlaudListAndReadRecording(t *testing.T) { + pc := newTestPlaud(t, newPlaudTestServer(), map[string]string{"tags": "voice, ideas"}) + recs, err := pc.ListRecordings(context.Background()) + if err != nil { + t.Fatal(err) + } + if len(recs) != 2 || recs[0].ID != "rec-2" || recs[1].ID != "rec-1" { + t.Fatalf("recs = %+v", recs) + } + if recs[1].Title != "Standup ideas" || recs[1].DurationMS != 125000 || len(recs[1].Tags) != 2 { + t.Fatalf("rec-1 = %+v", recs[1]) + } + if want := time.Date(2026, 11, 10, 8, 58, 0, 0, time.UTC); !recs[1].RecordedAt.Equal(want) { + t.Fatalf("recorded at = %v", recs[1].RecordedAt) + } + c, err := pc.ReadRecording(context.Background(), "rec-1") + if err != nil { + t.Fatal(err) + } + if !strings.Contains(c.Summary, "ship plaud connector") || !strings.Contains(c.Transcript, "Hello hive") { + t.Fatalf("content = %+v", c) + } + if _, err := pc.ReadRecording(context.Background(), "rec-2"); err == nil { + t.Fatal("recording with no note or transcript should error") + } +} diff --git a/src/pkg/knowledge/connector/syncer.go b/src/pkg/knowledge/connector/syncer.go index 160749e364..efe5efc6f7 100644 --- a/src/pkg/knowledge/connector/syncer.go +++ b/src/pkg/knowledge/connector/syncer.go @@ -159,6 +159,16 @@ func (s *Syncer) update(name string, fn func(*Status)) Status { return st } +// Connector returns the built connector for name; ok is false for an +// unknown name or one whose config failed to build. +func (s *Syncer) Connector(name string) (Connector, bool) { + e, ok := s.entries[name] + if !ok || e.buildErr != nil || e.conn == nil { + return nil, false + } + return e.conn, true +} + // SyncNow runs one sync of the named connector immediately, whether or not // it is enabled. It returns ErrSyncInProgress if a sync is already running. func (s *Syncer) SyncNow(ctx context.Context, name string) (Status, error) { diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11-1430-cursor.md b/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11-1430-cursor.md new file mode 100644 index 0000000000..edf494c416 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11-1430-cursor.md @@ -0,0 +1,3 @@ +# Dictated into Cursor + +Refactor the syncer so the wisprflow connector shares the plaud cursor parser. diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11.ndjson b/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11.ndjson new file mode 100644 index 0000000000..862be53254 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/2026-11-11.ndjson @@ -0,0 +1,3 @@ +{"id":"n-1","text":"Remember to add spek, hive and dibs to the Wispr dictionary.","at":"2026-11-11 08:30:00.000 +00:00","app":"md.obsidian","words":11} + +{"id":"n-2","text":"Hi","at":"2026-11-11 08:31:00.000 +00:00","app":"com.apple.MobileSMS","words":1} diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.csv b/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.csv new file mode 100644 index 0000000000..956dbdc67a --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.csv @@ -0,0 +1,2 @@ +id,text,timestamp,app,numWords +c-1,"Ask Greg about the KubeCon ops checklist, then file it in dibs.",2026-11-11T09:00:00Z,com.tinyspeck.slackmacgap,13 diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.json b/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.json new file mode 100644 index 0000000000..d9732572c6 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/dictations.json @@ -0,0 +1,4 @@ +{"dictations": [ + {"transcriptEntityId": "j-1", "formattedText": "Spec idea: a wisprflow knowledge connector for hive.", "asrText": "spec idea a wispr flow knowledge connector for hive", "timestamp": "2026-11-10T12:00:00Z", "app": "com.electron.wispr-flow", "numWords": 8}, + {"transcriptEntityId": "j-old", "formattedText": "Too old to matter.", "timestamp": "2019-05-05T12:00:00Z", "app": "com.apple.Notes"} +]} diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/ignored.bin b/src/pkg/knowledge/connector/testdata/wisprflow/export/ignored.bin new file mode 100644 index 0000000000..7c0582d8dd --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/ignored.bin @@ -0,0 +1 @@ +ignored binary diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/export/nested/frontmatter.md b/src/pkg/knowledge/connector/testdata/wisprflow/export/nested/frontmatter.md new file mode 100644 index 0000000000..0e1f51d3c3 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/export/nested/frontmatter.md @@ -0,0 +1,6 @@ +--- +id: m-fm +timestamp: 2026-11-10 16:20:00 +app: com.apple.Notes +--- +Front-matter note dictated into Apple Notes about the ACMM scorecard. diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/history.json b/src/pkg/knowledge/connector/testdata/wisprflow/history.json new file mode 100644 index 0000000000..633157e859 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/history.json @@ -0,0 +1,6 @@ +[{"id":"d-1","text":"Add a retry with backoff around the upload call.","raw":"um so add a retry with backoff around the upload call","at":"2026-11-10 09:15:00.000 +00:00","app":"com.microsoft.VSCode","url":"","duration":6.5,"numWords":9,"language":"en","status":"formatted","isArchived":0}, +{"id":"d-2","text":"Running late — start without me.","raw":"running late start without me","at":"2026-11-11 17:55:30.000 +00:00","app":"com.tinyspeck.slackmacgap","url":"","duration":2.1,"numWords":5,"language":"en","status":"formatted","isArchived":0}, +{"id":"d-short","text":"Ok.","raw":"ok","at":"2026-11-11 18:00:00.000 +00:00","app":"com.apple.MobileSMS","url":"","duration":0.8,"numWords":1,"language":"en","status":"formatted","isArchived":0}, +{"id":"d-raw","text":"still processing","raw":"still processing","at":"2026-11-11 18:05:00.000 +00:00","app":"com.google.Chrome","url":"","duration":3,"numWords":2,"language":"en","status":"raw_transcript","isArchived":0}, +{"id":"d-archived","text":"Archived thought about the hive dashboard.","raw":"archived thought about the hive dashboard","at":"2026-11-11 18:10:00.000 +00:00","app":"md.obsidian","url":"","duration":4,"numWords":6,"language":"en","status":"formatted","isArchived":1}, +{"id":"d-old","text":"Ancient dictation from long ago.","raw":"ancient dictation from long ago","at":"2020-01-01 10:00:00.000 +00:00","app":"com.google.Chrome","url":"https://example.com","duration":4,"numWords":5,"language":"en","status":"formatted","isArchived":0}] diff --git a/src/pkg/knowledge/connector/testdata/wisprflow/history.sql b/src/pkg/knowledge/connector/testdata/wisprflow/history.sql new file mode 100644 index 0000000000..1973358d97 --- /dev/null +++ b/src/pkg/knowledge/connector/testdata/wisprflow/history.sql @@ -0,0 +1,14 @@ +-- Minimal Wispr Flow History schema (community-documented, Sequelize DATETIME strings). +CREATE TABLE History ( + transcriptEntityId VARCHAR(36) PRIMARY KEY, + asrText TEXT, formattedText TEXT, editedText TEXT, + timestamp DATETIME, audio BLOB, screenshot BLOB, + status VARCHAR(255), app VARCHAR(255), url VARCHAR(255), + duration FLOAT, numWords INTEGER, language TEXT, isArchived TINYINT(1) DEFAULT 0 +); +INSERT INTO History VALUES ('d-1','um so add a retry with backoff around the upload call','Add a retry with backoff around the upload call.',NULL,'2026-11-10 09:15:00.000 +00:00',x'00',NULL,'formatted','com.microsoft.VSCode','',6.5,9,'en',0); +INSERT INTO History VALUES ('d-2','running late start without me','Running late, start without me.','Running late — start without me.','2026-11-11 17:55:30.000 +00:00',NULL,NULL,'formatted','com.tinyspeck.slackmacgap','',2.1,5,'en',0); +INSERT INTO History VALUES ('d-short','ok','Ok.',NULL,'2026-11-11 18:00:00.000 +00:00',NULL,NULL,'formatted','com.apple.MobileSMS','',0.8,1,'en',0); +INSERT INTO History VALUES ('d-raw','still processing','',NULL,'2026-11-11 18:05:00.000 +00:00',NULL,NULL,'raw_transcript','com.google.Chrome','',3,2,'en',0); +INSERT INTO History VALUES ('d-archived','archived thought about the hive dashboard','Archived thought about the hive dashboard.',NULL,'2026-11-11 18:10:00.000 +00:00',NULL,NULL,'formatted','md.obsidian','',4,6,'en',1); +INSERT INTO History VALUES ('d-old','ancient dictation from long ago','Ancient dictation from long ago.',NULL,'2020-01-01 10:00:00.000 +00:00',NULL,NULL,'formatted','com.google.Chrome','https://example.com',4,5,'en',0); diff --git a/src/pkg/knowledge/connector/wisprflow.go b/src/pkg/knowledge/connector/wisprflow.go new file mode 100644 index 0000000000..9ac4fc239c --- /dev/null +++ b/src/pkg/knowledge/connector/wisprflow.go @@ -0,0 +1,578 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "log/slog" + "os" + "os/exec" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "time" +) + +// TypeWisprFlow is the connector type for Wispr Flow dictation history, +// read from the desktop app's local SQLite file or from an export folder. +const TypeWisprFlow = "wisprflow" + +const ( + // wisprDefaultSQLite3 is the sqlite3 CLI used to read flow.sqlite; it + // ships with macOS and the hive image, so no Go SQLite driver is needed. + wisprDefaultSQLite3 = "sqlite3" + // wisprDefaultSinceDays bounds the first sync. + wisprDefaultSinceDays = 30 + // wisprDefaultMaxItems caps dictations read per sync. + wisprDefaultMaxItems = 500 + // wisprDefaultMinWords drops one- and two-word dictations ("yes", "ok"). + wisprDefaultMinWords = 3 + // wisprRelistOverlap re-reads this far behind the cursor: Flow updates a + // row in place (raw_transcript → formatted), so a late finalisation is + // still picked up. + wisprRelistOverlap = 2 * time.Hour + // wisprGroupDictation emits one fact per dictation; wisprGroupDay one + // fact per calendar day. + wisprGroupDictation = "dictation" + wisprGroupDay = "day" + // wisprTitleWords is how many words of the text make the fact title. + wisprTitleWords = 10 + // wisprSQLTimeout bounds one sqlite3 invocation. + wisprSQLTimeout = 60 * time.Second + // wisprStatusFormatted is the terminal row status in Flow's History table. + wisprStatusFormatted = "formatted" +) + +// wisprDictation is one Wispr Flow history row, normalised from SQLite or +// an export file. +type wisprDictation struct { + ID string + Text string + RawText string + At time.Time + App string + URL string + Duration float64 + Words int + Language string +} + +// wisprConnector reads Wispr Flow's local dictation history. Scope keys: +// path (flow.sqlite, a backup copy, or an export folder), sqlite3, since_days, +// max_items, min_words, group (dictation|day), apps, include_raw, tags. +// No auth: the data is already on local disk. +type wisprConnector struct { + cfg ConnectorConfig + logger *slog.Logger + now func() time.Time + // query runs the sqlite3 CLI; tests replace it. + query func(ctx context.Context, dbPath, sql string) ([]byte, error) +} + +func newWisprConnector(cfg ConnectorConfig, deps Deps) (Connector, error) { + c := &wisprConnector{cfg: cfg, logger: deps.logger(), now: time.Now} + bin := cfg.ScopeValue("sqlite3") + if bin == "" { + bin = wisprDefaultSQLite3 + } + c.query = wisprSQLite3Runner(bin) + return c, nil +} + +func (c *wisprConnector) Type() string { return TypeWisprFlow } + +// wisprDefaultPath is where the desktop app keeps its history on this OS. +func wisprDefaultPath() string { + switch runtime.GOOS { + case "darwin": + home, err := os.UserHomeDir() + if err != nil { + return "" + } + return filepath.Join(home, "Library", "Application Support", "Wispr Flow", "flow.sqlite") + case "windows": + if appdata := os.Getenv("APPDATA"); appdata != "" { + return filepath.Join(appdata, "Wispr Flow", "flow.sqlite") + } + } + return "" +} + +func (c *wisprConnector) path() string { + if p := c.cfg.ScopeValue("path"); p != "" { + return p + } + return wisprDefaultPath() +} + +// Validate checks the scope. The path may not exist yet (the app may be +// installed later); only its shape is checked. +func (c *wisprConnector) Validate(cfg ConnectorConfig) error { + if cfg.Auth.Configured() { + return fmt.Errorf("wisprflow: auth.env/auth.file are not used; the connector reads Wispr Flow's local history file") + } + p := cfg.ScopeValue("path") + if p == "" && wisprDefaultPath() == "" { + return fmt.Errorf("wisprflow: scope.path is required on %s (no default Wispr Flow location)", runtime.GOOS) + } + if p != "" && !filepath.IsAbs(p) { + return fmt.Errorf("wisprflow: scope.path must be an absolute path to flow.sqlite or an export folder") + } + for _, key := range []string{"since_days", "max_items", "min_words"} { + if v := cfg.ScopeValue(key); v != "" { + n, err := strconv.Atoi(v) + if err != nil || n < 0 || (key != "min_words" && n == 0) { + return fmt.Errorf("wisprflow: scope.%s must be a positive integer", key) + } + } + } + if g := cfg.ScopeValue("group"); g != "" && g != wisprGroupDictation && g != wisprGroupDay { + return fmt.Errorf("wisprflow: scope.group must be %q or %q", wisprGroupDictation, wisprGroupDay) + } + if v := cfg.ScopeValue("include_raw"); v != "" { + if _, err := strconv.ParseBool(v); err != nil { + return fmt.Errorf("wisprflow: scope.include_raw must be true or false") + } + } + if bin := cfg.ScopeValue("sqlite3"); bin != "" { + if _, err := splitCommandLine(bin); err != nil { + return err + } + } + return nil +} + +func (c *wisprConnector) scopeInt(key string, def int) int { + v := c.cfg.ScopeValue(key) + if v == "" { + return def + } + n, err := strconv.Atoi(v) + if err != nil || n < 0 { + return def + } + return n +} + +func (c *wisprConnector) group() string { + if c.cfg.ScopeValue("group") == wisprGroupDay { + return wisprGroupDay + } + return wisprGroupDictation +} + +// Sync reads dictations newer than the cursor (minus an overlap) and emits +// one page per dictation or per day. The listing is incremental, so unlisted +// pages are never tombstoned. +func (c *wisprConnector) Sync(ctx context.Context, cur Cursor, emit func(Page) error) (Cursor, error) { + p := c.path() + if p == "" { + return cur, fmt.Errorf("wisprflow: no path configured") + } + info, err := os.Stat(p) + if err != nil { + return cur, fmt.Errorf("wisprflow: %w (install Wispr Flow, point scope.path at flow.sqlite or an export folder, or grant the hive process Full Disk Access on macOS)", err) + } + since := parsePlaudCursor(cur) + if since.IsZero() { + since = c.now().Add(-time.Duration(c.scopeInt("since_days", wisprDefaultSinceDays)) * 24 * time.Hour) + } else { + since = since.Add(-wisprRelistOverlap) + } + if c.group() == wisprGroupDay { + // A day page is rebuilt whole, so re-read from the start of that day. + y, m, d := since.UTC().Date() + since = time.Date(y, m, d, 0, 0, 0, 0, time.UTC) + } + maxItems := c.scopeInt("max_items", wisprDefaultMaxItems) + + var rows []wisprDictation + var partial error + if info.IsDir() { + rows, partial = readWisprExportDir(p, since, maxItems) + } else { + rows, err = c.readSQLite(ctx, p, since, maxItems) + if err != nil { + return cur, err + } + } + rows = c.filter(rows) + sort.Slice(rows, func(i, j int) bool { return rows[i].At.Before(rows[j].At) }) + + newest := parsePlaudCursor(cur) + for _, r := range rows { + if r.At.After(newest) { + newest = r.At + } + } + pages := c.pages(rows) + for _, pg := range pages { + if err := ctx.Err(); err != nil { + return cur, err + } + if err := emit(pg); err != nil { + return cur, err + } + } + next := cur + if !newest.IsZero() { + next = Cursor(newest.UTC().Format(time.RFC3339)) + } + if partial != nil { + return next, &PartialError{Err: partial} + } + return next, nil +} + +// filter applies min_words and the apps allow-list. +func (c *wisprConnector) filter(rows []wisprDictation) []wisprDictation { + minWords := c.scopeInt("min_words", wisprDefaultMinWords) + var apps []string + for _, a := range strings.Split(c.cfg.ScopeValue("apps"), ",") { + if a = strings.ToLower(strings.TrimSpace(a)); a != "" { + apps = append(apps, a) + } + } + out := rows[:0] + for _, r := range rows { + r.Text = strings.TrimSpace(r.Text) + if r.Text == "" || r.ID == "" { + continue + } + words := r.Words + if words <= 0 { + words = len(strings.Fields(r.Text)) + } + if words < minWords { + continue + } + if len(apps) > 0 && !wisprAppMatches(r.App, apps) { + continue + } + out = append(out, r) + } + return out +} + +func wisprAppMatches(app string, allow []string) bool { + app = strings.ToLower(app) + for _, a := range allow { + if strings.Contains(app, a) { + return true + } + } + return false +} + +// pages renders one page per dictation, or one per UTC day. +func (c *wisprConnector) pages(rows []wisprDictation) []Page { + includeRaw, _ := strconv.ParseBool(c.cfg.ScopeValue("include_raw")) + tags := c.cfg.ScopeValue("tags") + if c.group() == wisprGroupDictation { + out := make([]Page, 0, len(rows)) + for _, r := range rows { + out = append(out, c.dictationPage(r, includeRaw, tags)) + } + return out + } + byDay := map[string][]wisprDictation{} + var days []string + for _, r := range rows { + day := r.At.UTC().Format("2006-01-02") + if _, ok := byDay[day]; !ok { + days = append(days, day) + } + byDay[day] = append(byDay[day], r) + } + sort.Strings(days) + out := make([]Page, 0, len(days)) + for _, day := range days { + out = append(out, c.dayPage(day, byDay[day], includeRaw, tags)) + } + return out +} + +func wisprTitle(text string) string { + words := strings.Fields(text) + if len(words) > wisprTitleWords { + return strings.Join(words[:wisprTitleWords], " ") + "…" + } + return strings.Join(words, " ") +} + +// wisprAppName turns a bundle id like com.tinyspeck.slackmacgap into a +// readable label; unknown ids keep their last segment. +func wisprAppName(app string) string { + app = strings.TrimSpace(app) + if app == "" { + return "" + } + known := map[string]string{ + "com.tinyspeck.slackmacgap": "Slack", + "md.obsidian": "Obsidian", + "com.apple.MobileSMS": "Messages", + "com.microsoft.VSCode": "VS Code", + "com.google.Chrome": "Chrome", + "com.apple.Safari": "Safari", + "com.electron.wispr-flow": "Wispr Flow", + "com.openai.chat": "ChatGPT", + "com.anthropic.claudefordesktop": "Claude", + "com.apple.Notes": "Notes", + "com.apple.mail": "Mail", + "com.apple.Terminal": "Terminal", + "com.googlecode.iterm2": "iTerm2", + "com.todesktop.230313mzl4w4u92": "Cursor", + } + if n, ok := known[app]; ok { + return n + } + if i := strings.LastIndex(app, "."); i >= 0 && i < len(app)-1 { + return app[i+1:] + } + return app +} + +func (c *wisprConnector) dictationPage(r wisprDictation, includeRaw bool, tags string) Page { + appName := wisprAppName(r.App) + title := wisprTitle(r.Text) + var md strings.Builder + fmt.Fprintf(&md, "# %s\n\n", title) + fmt.Fprintf(&md, "- **Dictated:** %s\n", r.At.UTC().Format(time.RFC3339)) + if appName != "" { + fmt.Fprintf(&md, "- **App:** %s\n", appName) + } + if r.URL != "" { + fmt.Fprintf(&md, "- **URL:** %s\n", r.URL) + } + if r.Duration > 0 { + fmt.Fprintf(&md, "- **Duration:** %s\n", (time.Duration(r.Duration * float64(time.Second))).Round(time.Second)) + } + md.WriteString("\n## Transcript\n\n") + md.WriteString(r.Text) + md.WriteString("\n") + if includeRaw && strings.TrimSpace(r.RawText) != "" && strings.TrimSpace(r.RawText) != r.Text { + md.WriteString("\n## Raw transcript\n\n") + md.WriteString(strings.TrimSpace(r.RawText)) + md.WriteString("\n") + } + attrs := map[string]string{ + "wispr_transcript_id": r.ID, + "dictated_at": r.At.UTC().Format(time.RFC3339), + "recorded_at": r.At.UTC().Format(time.RFC3339), + } + if r.App != "" { + attrs["app"] = r.App + } + if r.Words > 0 { + attrs["num_words"] = strconv.Itoa(r.Words) + } + if r.Language != "" { + attrs["language"] = r.Language + } + if tags != "" { + attrs["tags"] = tags + } + return Page{ + ID: r.ID, + Title: title, + URL: r.URL, + Markdown: md.String(), + UpdatedAt: r.At, + Path: []string{"Wispr Flow", r.At.UTC().Format("2006-01-02")}, + Attrs: attrs, + } +} + +func (c *wisprConnector) dayPage(day string, rows []wisprDictation, includeRaw bool, tags string) Page { + title := "Wispr Flow dictations " + day + var md strings.Builder + fmt.Fprintf(&md, "# %s\n\n", title) + fmt.Fprintf(&md, "%d dictation(s).\n\n", len(rows)) + var newest time.Time + words := 0 + for _, r := range rows { + if r.At.After(newest) { + newest = r.At + } + words += r.Words + label := r.At.UTC().Format("15:04") + if n := wisprAppName(r.App); n != "" { + label += " · " + n + } + fmt.Fprintf(&md, "## %s\n\n%s\n\n", label, r.Text) + if includeRaw && strings.TrimSpace(r.RawText) != "" && strings.TrimSpace(r.RawText) != r.Text { + fmt.Fprintf(&md, "<details><summary>Raw transcript</summary>\n\n%s\n\n</details>\n\n", strings.TrimSpace(r.RawText)) + } + } + attrs := map[string]string{"day": day, "dictations": strconv.Itoa(len(rows)), "recorded_at": newest.UTC().Format(time.RFC3339)} + if words > 0 { + attrs["num_words"] = strconv.Itoa(words) + } + if tags != "" { + attrs["tags"] = tags + } + return Page{ + ID: "day-" + day, + Title: title, + Markdown: md.String(), + UpdatedAt: newest, + Path: []string{"Wispr Flow"}, + Attrs: attrs, + } +} + +// wisprSQLite3Runner executes the sqlite3 CLI read-only with JSON output. +// The History table holds audio and screenshot BLOBs, so only text columns +// are ever selected. +func wisprSQLite3Runner(bin string) func(ctx context.Context, dbPath, sql string) ([]byte, error) { + return func(ctx context.Context, dbPath, sql string) ([]byte, error) { + argv, err := splitCommandLine(bin) + if err != nil { + return nil, err + } + ctx, cancel := context.WithTimeout(ctx, wisprSQLTimeout) + defer cancel() + args := append(argv[1:], "-readonly", "-json", "-bail", dbPath, sql) + cmd := exec.CommandContext(ctx, argv[0], args...) + out, err := cmd.Output() + if err != nil { + var stderr string + if ee, ok := err.(*exec.ExitError); ok { + stderr = firstLine(string(ee.Stderr)) + } + if stderr == "" { + stderr = err.Error() + } + return nil, fmt.Errorf("wisprflow: sqlite3 failed: %s", stderr) + } + return out, nil + } +} + +// wisprHistorySQL selects finalised, unarchived dictations newer than since. +// Flow stores Sequelize DATETIME strings; datetime() normalises them so the +// comparison does not depend on the stored format. +func wisprHistorySQL(since time.Time, limit int) string { + s := strings.ReplaceAll(since.UTC().Format("2006-01-02 15:04:05"), "'", "''") + return fmt.Sprintf(`SELECT transcriptEntityId AS id, `+ + `COALESCE(NULLIF(editedText,''), NULLIF(formattedText,''), asrText) AS text, `+ + `asrText AS raw, timestamp AS at, COALESCE(app,'') AS app, COALESCE(url,'') AS url, `+ + `COALESCE(duration,0) AS duration, COALESCE(numWords,0) AS numWords, COALESCE(language,'') AS language `+ + `FROM History WHERE COALESCE(isArchived,0)=0 AND status='%s' AND datetime(timestamp) > datetime('%s') `+ + `ORDER BY datetime(timestamp) ASC LIMIT %d;`, wisprStatusFormatted, s, limit) +} + +func (c *wisprConnector) readSQLite(ctx context.Context, dbPath string, since time.Time, limit int) ([]wisprDictation, error) { + out, err := c.query(ctx, dbPath, wisprHistorySQL(since, limit)) + if err != nil { + return nil, err + } + return decodeWisprJSON(out) +} + +// decodeWisprJSON accepts the sqlite3 -json array (or any JSON array/object +// list of dictations, as written by community exporters). +func decodeWisprJSON(data []byte) ([]wisprDictation, error) { + text := strings.TrimSpace(string(data)) + if text == "" { + return nil, nil + } + var raw any + if err := json.Unmarshal([]byte(text), &raw); err != nil { + return nil, fmt.Errorf("wisprflow: invalid JSON: %s", firstLine(text)) + } + var items []any + switch v := raw.(type) { + case []any: + items = v + case map[string]any: + for _, key := range []string{"dictations", "history", "items", "data", "results", "transcripts"} { + if arr, ok := v[key].([]any); ok { + items = arr + break + } + } + if items == nil { + items = []any{v} + } + } + out := make([]wisprDictation, 0, len(items)) + for _, it := range items { + m, ok := it.(map[string]any) + if !ok { + continue + } + if d, ok := wisprFromMap(m); ok { + out = append(out, d) + } + } + return out, nil +} + +// wisprFromMap normalises one dictation object from SQLite JSON or an +// exporter's JSON, which name the same fields differently. +func wisprFromMap(m map[string]any) (wisprDictation, bool) { + d := wisprDictation{ + ID: plaudString(m, "id", "transcriptEntityId", "transcript_id", "uuid"), + Text: plaudString(m, "text", "editedText", "edited_text", "formattedText", "formatted_text", "content", "asrText"), + RawText: plaudString(m, "raw", "asrText", "asr_text", "raw_text"), + At: plaudTime(m, "at", "timestamp", "created_at", "createdAt", "date"), + App: plaudString(m, "app", "application", "bundle_id"), + URL: plaudString(m, "url"), + Language: plaudString(m, "language", "lang"), + } + d.Duration = wisprFloat(m, "duration", "duration_s", "seconds") + d.Words = int(wisprFloat(m, "numWords", "num_words", "words")) + if d.At.IsZero() { + d.At = wisprParseTime(plaudString(m, "at", "timestamp", "created_at", "createdAt", "date")) + } + if d.ID == "" || strings.TrimSpace(d.Text) == "" || d.At.IsZero() { + return d, false + } + return d, true +} + +// wisprParseTime handles Sequelize's "2026-11-11 14:05:09.123 +00:00" and +// the common ISO shapes; plaudTime covers RFC3339 and epoch numbers. +func wisprParseTime(s string) time.Time { + s = strings.TrimSpace(s) + if s == "" { + return time.Time{} + } + for _, layout := range []string{ + "2006-01-02 15:04:05.000 -07:00", + "2006-01-02 15:04:05 -07:00", + "2006-01-02 15:04:05.000Z07:00", + "2006-01-02 15:04:05.000", + "2006-01-02 15:04:05", + "2006-01-02T15:04:05.000Z07:00", + time.RFC3339Nano, + time.RFC3339, + "2006-01-02T15:04", + "2006-01-02 15:04", + "2006-01-02", + } { + if t, err := time.Parse(layout, s); err == nil { + return t.UTC() + } + } + return time.Time{} +} + +// wisprFloat reads the first numeric value among keys, accepting JSON +// numbers and numeric strings (CSV exports). +func wisprFloat(m map[string]any, keys ...string) float64 { + for _, k := range keys { + switch v := m[k].(type) { + case float64: + return v + case string: + if f, err := strconv.ParseFloat(strings.TrimSpace(v), 64); err == nil { + return f + } + } + } + return 0 +} diff --git a/src/pkg/knowledge/connector/wisprflow_export.go b/src/pkg/knowledge/connector/wisprflow_export.go new file mode 100644 index 0000000000..3a6d592b15 --- /dev/null +++ b/src/pkg/knowledge/connector/wisprflow_export.go @@ -0,0 +1,259 @@ +package connector + +import ( + "bufio" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "regexp" + "strings" + "time" +) + +// wisprExportMaxFileBytes caps one export file read. +const wisprExportMaxFileBytes = 64 << 20 + +// wisprFilenameTimeRe pulls a date (and optional time) from an exported +// markdown filename such as 2026-11-11-1405-slack.md or 2026-11-11T14-05-09.md. +var wisprFilenameTimeRe = regexp.MustCompile(`(\d{4}-\d{2}-\d{2})(?:[T _-](\d{2})[-:h]?(\d{2})(?:[-:m]?(\d{2}))?)?`) + +// readWisprExportDir walks an export folder (as written by `wispr export`, +// wispr-flow-exporter or a hand-made dump) and returns the dictations newer +// than since. Supported files: .json (array or wrapped object), .ndjson / +// .jsonl (one object per line), .csv (header row) and .md (one dictation per +// file, optional YAML front matter). Unreadable files are reported together +// as one error while the readable ones are still returned. +func readWisprExportDir(dir string, since time.Time, limit int) ([]wisprDictation, error) { + var out []wisprDictation + var problems []string + seen := map[string]bool{} + err := filepath.WalkDir(dir, func(path string, d fs.DirEntry, walkErr error) error { + if walkErr != nil { + problems = append(problems, fmt.Sprintf("%s: %v", path, walkErr)) + return nil + } + name := d.Name() + if d.IsDir() { + if path != dir && strings.HasPrefix(name, ".") { + return filepath.SkipDir + } + return nil + } + if strings.HasPrefix(name, ".") { + return nil + } + rows, err := readWisprExportFile(dir, path) + if err != nil { + problems = append(problems, fmt.Sprintf("%s: %v", filepath.Base(path), err)) + return nil + } + for _, r := range rows { + if r.At.IsZero() || !r.At.After(since) || seen[r.ID] { + continue + } + seen[r.ID] = true + out = append(out, r) + if len(out) >= limit { + return fs.SkipAll + } + } + return nil + }) + if err != nil { + return out, err + } + if len(problems) > 0 { + return out, fmt.Errorf("wisprflow: %d export file(s) skipped: %s", len(problems), strings.Join(problems, "; ")) + } + return out, nil +} + +// readWisprExportFile dispatches on extension; unknown extensions are ignored. +func readWisprExportFile(root, path string) ([]wisprDictation, error) { + ext := strings.ToLower(filepath.Ext(path)) + switch ext { + case ".json", ".ndjson", ".jsonl", ".csv", ".md", ".markdown", ".txt": + default: + return nil, nil + } + info, err := os.Stat(path) + if err != nil { + return nil, err + } + if info.Size() > wisprExportMaxFileBytes { + return nil, fmt.Errorf("file larger than %d bytes", wisprExportMaxFileBytes) + } + data, err := os.ReadFile(path) + if err != nil { + return nil, err + } + switch ext { + case ".json": + return decodeWisprJSON(data) + case ".ndjson", ".jsonl": + return decodeWisprNDJSON(data) + case ".csv": + return decodeWisprCSV(data) + default: + rel, relErr := filepath.Rel(root, path) + if relErr != nil { + rel = filepath.Base(path) + } + d, ok := decodeWisprMarkdown(rel, data, info.ModTime()) + if !ok { + return nil, nil + } + return []wisprDictation{d}, nil + } +} + +func decodeWisprNDJSON(data []byte) ([]wisprDictation, error) { + var out []wisprDictation + sc := bufio.NewScanner(strings.NewReader(string(data))) + sc.Buffer(make([]byte, 0, 64<<10), wisprExportMaxFileBytes) + line := 0 + for sc.Scan() { + line++ + text := strings.TrimSpace(sc.Text()) + if text == "" { + continue + } + var m map[string]any + if err := json.Unmarshal([]byte(text), &m); err != nil { + return out, fmt.Errorf("line %d: %v", line, err) + } + if d, ok := wisprFromMap(m); ok { + out = append(out, d) + } + } + return out, sc.Err() +} + +// decodeWisprCSV maps header names onto the same keys wisprFromMap accepts. +func decodeWisprCSV(data []byte) ([]wisprDictation, error) { + r := csv.NewReader(strings.NewReader(string(data))) + r.FieldsPerRecord = -1 + r.LazyQuotes = true + header, err := r.Read() + if err != nil { + if errors.Is(err, io.EOF) { + return nil, nil + } + return nil, err + } + for i := range header { + header[i] = strings.TrimSpace(strings.TrimPrefix(header[i], "\ufeff")) + } + var out []wisprDictation + for { + rec, err := r.Read() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return out, err + } + m := make(map[string]any, len(header)) + for i, h := range header { + if i < len(rec) && h != "" { + m[h] = rec[i] + } + } + if d, ok := wisprFromMap(m); ok { + out = append(out, d) + } + } + return out, nil +} + +// decodeWisprMarkdown reads one dictation per markdown file. Front matter +// keys id, timestamp/date/created, app and url are honoured; otherwise the +// id is the relative path and the time comes from the filename or mtime. +func decodeWisprMarkdown(rel string, data []byte, modTime time.Time) (wisprDictation, bool) { + body := strings.ReplaceAll(string(data), "\r\n", "\n") + meta := map[string]string{} + if strings.HasPrefix(body, "---\n") { + if end := strings.Index(body[4:], "\n---"); end >= 0 { + for _, line := range strings.Split(body[4:4+end], "\n") { + k, v, ok := strings.Cut(line, ":") + if !ok { + continue + } + meta[strings.ToLower(strings.TrimSpace(k))] = strings.Trim(strings.TrimSpace(v), `"'`) + } + body = body[4+end+4:] + } + } + body = strings.TrimSpace(body) + // Drop a leading heading: the title is derived from the text. + if strings.HasPrefix(body, "# ") { + if i := strings.IndexByte(body, '\n'); i >= 0 { + body = strings.TrimSpace(body[i+1:]) + } else { + body = "" + } + } + d := wisprDictation{ + Text: body, + App: meta["app"], + URL: meta["url"], + Language: meta["language"], + } + d.ID = meta["id"] + if d.ID == "" { + d.ID = meta["transcriptentityid"] + } + if d.ID == "" { + d.ID = strings.TrimSuffix(filepath.ToSlash(rel), filepath.Ext(rel)) + } + for _, k := range []string{"timestamp", "dictated", "date", "created", "created_at", "createdat"} { + if v := meta[k]; v != "" { + d.At = wisprParseTime(v) + if d.At.IsZero() { + if t, ok := wisprTimeFromName(v); ok { + d.At = t + } + } + break + } + } + if d.At.IsZero() { + if t, ok := wisprTimeFromName(filepath.Base(rel)); ok { + d.At = t + } else { + d.At = modTime.UTC() + } + } + if d.Text == "" { + return d, false + } + return d, true +} + +// wisprTimeFromName extracts a UTC time from a filename-style date stamp. +func wisprTimeFromName(name string) (time.Time, bool) { + m := wisprFilenameTimeRe.FindStringSubmatch(name) + if m == nil { + return time.Time{}, false + } + stamp := m[1] + layout := "2006-01-02" + if m[2] != "" { + stamp += " " + m[2] + ":" + m[3] + layout = "2006-01-02 15:04" + if m[4] != "" { + stamp += ":" + m[4] + layout = "2006-01-02 15:04:05" + } + } + t, err := time.Parse(layout, stamp) + if err != nil { + return time.Time{}, false + } + return t.UTC(), true +} diff --git a/src/pkg/knowledge/connector/wisprflow_test.go b/src/pkg/knowledge/connector/wisprflow_test.go new file mode 100644 index 0000000000..64545e2abd --- /dev/null +++ b/src/pkg/knowledge/connector/wisprflow_test.go @@ -0,0 +1,378 @@ +package connector + +import ( + "context" + "encoding/json" + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + "testing" + "time" +) + +func wisprTestNow() time.Time { return time.Date(2026, 11, 11, 20, 0, 0, 0, time.UTC) } + +func newTestWispr(t *testing.T, scope map[string]string) *wisprConnector { + t.Helper() + cfg := ConnectorConfig{Name: "wispr", Type: TypeWisprFlow, Enabled: true, Layer: "personal", Scope: scope} + c, err := DefaultRegistry().New(cfg, Deps{StateDir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + wc := c.(*wisprConnector) + wc.now = wisprTestNow + return wc +} + +// wisprFixtureDB builds flow.sqlite from testdata/wisprflow/history.sql with +// the sqlite3 CLI the connector itself shells out to; callers skip when the +// CLI is absent (CI runners), so the sync tests use wisprFakeQuery instead. +func wisprFixtureDB(t *testing.T) string { + t.Helper() + if _, err := exec.LookPath("sqlite3"); err != nil { + t.Skip("sqlite3 CLI not installed") + } + sql, err := os.ReadFile(filepath.Join("testdata", "wisprflow", "history.sql")) + if err != nil { + t.Fatal(err) + } + db := filepath.Join(t.TempDir(), "flow.sqlite") + cmd := exec.Command("sqlite3", db) + cmd.Stdin = strings.NewReader(string(sql)) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("building fixture db: %v: %s", err, out) + } + return db +} + +var wisprSinceRE = regexp.MustCompile(`datetime\('([^']+)'\)`) + +// wisprFakeQuery stands in for the sqlite3 CLI: it serves the rows of +// testdata/wisprflow/history.json (the same data as history.sql) and applies +// the WHERE clause of wisprHistorySQL — status, isArchived and the since +// bound — so the connector sees exactly what sqlite3 -json would return. +func wisprFakeQuery(t *testing.T) func(ctx context.Context, dbPath, sql string) ([]byte, error) { + t.Helper() + data, err := os.ReadFile(filepath.Join("testdata", "wisprflow", "history.json")) + if err != nil { + t.Fatal(err) + } + var rows []map[string]any + if err := json.Unmarshal(data, &rows); err != nil { + t.Fatal(err) + } + return func(_ context.Context, dbPath, sql string) ([]byte, error) { + if _, err := os.Stat(dbPath); err != nil { + return nil, fmt.Errorf("wisprflow: sqlite3 failed: unable to open database") + } + m := wisprSinceRE.FindStringSubmatch(sql) + if m == nil || !strings.Contains(sql, "status='formatted'") { + return nil, fmt.Errorf("unexpected sql: %s", sql) + } + since, err := time.Parse("2006-01-02 15:04:05", m[1]) + if err != nil { + return nil, err + } + var out []map[string]any + for _, r := range rows { + at := wisprParseTime(r["at"].(string)) + if r["status"] != "formatted" || r["isArchived"] != float64(0) || !at.After(since) { + continue + } + out = append(out, r) + } + return json.Marshal(out) + } +} + +// newFakeWispr returns a connector whose sqlite3 runner is wisprFakeQuery +// and whose path points at an existing (empty) flow.sqlite file. +func newFakeWispr(t *testing.T, scope map[string]string) *wisprConnector { + t.Helper() + db := filepath.Join(t.TempDir(), "flow.sqlite") + if err := os.WriteFile(db, nil, 0o600); err != nil { + t.Fatal(err) + } + if scope == nil { + scope = map[string]string{} + } + scope["path"] = db + c := newTestWispr(t, scope) + c.query = wisprFakeQuery(t) + return c +} + +func collectWispr(t *testing.T, c *wisprConnector, cur Cursor) ([]Page, Cursor, error) { + t.Helper() + var pages []Page + next, err := c.Sync(context.Background(), cur, func(p Page) error { + pages = append(pages, p) + return nil + }) + return pages, next, err +} + +func wisprIDs(pages []Page) []string { + out := make([]string, 0, len(pages)) + for _, p := range pages { + out = append(out, p.ID) + } + return out +} + +func TestWisprValidate(t *testing.T) { + c := newTestWispr(t, map[string]string{"path": "/x/flow.sqlite"}) + cases := []struct { + name string + cfg ConnectorConfig + want string + }{ + {"auth rejected", ConnectorConfig{Auth: Auth{Env: "X"}, Scope: map[string]string{"path": "/x/flow.sqlite"}}, "not used"}, + {"relative path", ConnectorConfig{Scope: map[string]string{"path": "flow.sqlite"}}, "absolute"}, + {"bad group", ConnectorConfig{Scope: map[string]string{"path": "/x", "group": "week"}}, "scope.group"}, + {"bad since", ConnectorConfig{Scope: map[string]string{"path": "/x", "since_days": "0"}}, "since_days"}, + {"bad include_raw", ConnectorConfig{Scope: map[string]string{"path": "/x", "include_raw": "maybe"}}, "include_raw"}, + } + for _, tc := range cases { + err := c.Validate(tc.cfg) + if err == nil || !strings.Contains(err.Error(), tc.want) { + t.Errorf("%s: err = %v, want containing %q", tc.name, err, tc.want) + } + } + ok := ConnectorConfig{Scope: map[string]string{"path": "/x/flow.sqlite", "group": "day", "since_days": "7", "min_words": "0", "include_raw": "true"}} + if err := c.Validate(ok); err != nil { + t.Fatalf("valid config rejected: %v", err) + } +} + +func TestWisprSyncSQLite(t *testing.T) { + c := newFakeWispr(t, map[string]string{"tags": "voice", "include_raw": "true"}) + + pages, cur, err := collectWispr(t, c, "") + if err != nil { + t.Fatal(err) + } + // d-short (1 word), d-raw (not formatted), d-archived and d-old (before + // since_days) are all excluded. + if got := wisprIDs(pages); strings.Join(got, ",") != "d-1,d-2" { + t.Fatalf("pages = %v", got) + } + if cur != "2026-11-11T17:55:30Z" { + t.Fatalf("cursor = %q", cur) + } + p := pages[1] + if p.Title != "Running late — start without me." { + t.Errorf("title = %q (editedText should win)", p.Title) + } + for _, want := range []string{"**App:** Slack", "**Dictated:** 2026-11-11T17:55:30Z", "## Raw transcript", "running late start without me"} { + if !strings.Contains(p.Markdown, want) { + t.Errorf("markdown missing %q:\n%s", want, p.Markdown) + } + } + if p.Attrs["wispr_transcript_id"] != "d-2" || p.Attrs["app"] != "com.tinyspeck.slackmacgap" || p.Attrs["tags"] != "voice" || p.Attrs["num_words"] != "5" { + t.Errorf("attrs = %v", p.Attrs) + } + if strings.Join(p.Path, "/") != "Wispr Flow/2026-11-11" { + t.Errorf("path = %v", p.Path) + } + if _, ok := pages[0].Attrs["tags"]; !ok || strings.Contains(pages[0].Markdown, "## Raw transcript") == false { + t.Errorf("first page attrs/raw = %v", pages[0].Attrs) + } + + // Incremental: the overlap window re-reads d-2 only. + pages, cur2, err := collectWispr(t, c, cur) + if err != nil { + t.Fatal(err) + } + if got := wisprIDs(pages); strings.Join(got, ",") != "d-2" { + t.Fatalf("incremental pages = %v", got) + } + if cur2 != cur { + t.Fatalf("cursor moved to %q", cur2) + } +} + +func TestWisprFactInboxMetadata(t *testing.T) { + c := newFakeWispr(t, map[string]string{"include_raw": "true"}) + pages, _, err := collectWispr(t, c, "") + if err != nil { + t.Fatal(err) + } + w := testWriter(t) + slug, _, err := w.Write(c.cfg, pages[1]) + if err != nil { + t.Fatal(err) + } + data, err := os.ReadFile(filepath.Join(w.Dir, slug+".md")) + if err != nil { + t.Fatal(err) + } + fact := string(data) + for _, want := range []string{ + "source: wisprflow\n", + "source_id: d-2\n", + "attr_recorded_at: 2026-11-11T17:55:30Z\n", + "## Transcript\n\nRunning late — start without me.\n", + } { + if !strings.Contains(fact, want) { + t.Errorf("fact missing %q:\n%s", want, fact) + } + } +} + +func TestWisprSyncAppsFilterAndDayGroup(t *testing.T) { + c := newFakeWispr(t, map[string]string{"apps": "slack, vscode", "group": "day", "since_days": "3650"}) + pages, _, err := collectWispr(t, c, "") + if err != nil { + t.Fatal(err) + } + if got := wisprIDs(pages); strings.Join(got, ",") != "day-2026-11-10,day-2026-11-11" { + t.Fatalf("pages = %v", got) + } + day := pages[1] + if day.Attrs["dictations"] != "1" || day.Attrs["recorded_at"] != "2026-11-11T17:55:30Z" || !strings.Contains(day.Markdown, "## 17:55 · Slack") { + t.Errorf("day page = %v\n%s", day.Attrs, day.Markdown) + } + if strings.Contains(day.Markdown, "Ok.") { + t.Error("one-word dictation leaked into the day page") + } + // d-old is Chrome, so even with a 10-year window the apps filter drops it. + for _, p := range pages { + if strings.Contains(p.Markdown, "Ancient") { + t.Error("apps filter did not drop the Chrome dictation") + } + } +} + +// TestWisprSQLite3CLI runs the real sqlite3 CLI against the fixture database +// and checks it agrees with the fake used by the other tests. +func TestWisprSQLite3CLI(t *testing.T) { + db := wisprFixtureDB(t) + c := newTestWispr(t, map[string]string{"path": db, "since_days": "3650"}) + pages, cur, err := collectWispr(t, c, "") + if err != nil { + t.Fatal(err) + } + if got := wisprIDs(pages); strings.Join(got, ",") != "d-old,d-1,d-2" { + t.Fatalf("pages = %v", got) + } + if cur != "2026-11-11T17:55:30Z" { + t.Fatalf("cursor = %q", cur) + } +} + +func TestWisprSyncMissingPath(t *testing.T) { + c := newTestWispr(t, map[string]string{"path": filepath.Join(t.TempDir(), "nope", "flow.sqlite")}) + _, cur, err := collectWispr(t, c, "2026-01-01T00:00:00Z") + if err == nil || !strings.Contains(err.Error(), "Full Disk Access") { + t.Fatalf("err = %v", err) + } + if cur != "2026-01-01T00:00:00Z" { + t.Fatalf("cursor changed on error: %q", cur) + } +} + +func TestWisprSyncSQLiteError(t *testing.T) { + abs, _ := filepath.Abs(filepath.Join("testdata", "wisprflow", "history.sql")) + c := newTestWispr(t, map[string]string{"path": abs}) + c.query = func(context.Context, string, string) ([]byte, error) { + return nil, &PartialError{Err: os.ErrPermission} + } + if _, _, err := collectWispr(t, c, ""); err == nil { + t.Fatal("expected sqlite error to surface") + } +} + +func TestWisprHistorySQL(t *testing.T) { + sql := wisprHistorySQL(time.Date(2026, 11, 1, 12, 0, 0, 0, time.UTC), 7) + for _, want := range []string{"status='formatted'", "datetime('2026-11-01 12:00:00')", "LIMIT 7", "COALESCE(isArchived,0)=0", "NULLIF(editedText,'')"} { + if !strings.Contains(sql, want) { + t.Errorf("sql missing %q: %s", want, sql) + } + } + if strings.Contains(sql, "audio") || strings.Contains(sql, "screenshot") { + t.Error("sql must not select BLOB columns") + } +} + +func TestWisprSyncExportDir(t *testing.T) { + dir, _ := filepath.Abs(filepath.Join("testdata", "wisprflow", "export")) + c := newTestWispr(t, map[string]string{"path": dir}) + pages, cur, err := collectWispr(t, c, "") + if err != nil { + t.Fatal(err) + } + // j-old is before since_days; n-2 is one word; ignored.bin is skipped. + if got := wisprIDs(pages); strings.Join(got, ",") != "j-1,m-fm,n-1,c-1,2026-11-11-1430-cursor" { + t.Fatalf("pages = %v", got) + } + if cur != "2026-11-11T14:30:00Z" { + t.Fatalf("cursor = %q", cur) + } + byID := map[string]Page{} + for _, p := range pages { + byID[p.ID] = p + } + if p := byID["2026-11-11-1430-cursor"]; !strings.HasPrefix(p.Title, "Refactor the syncer") || strings.Contains(p.Markdown, "# Dictated into Cursor") { + t.Errorf("markdown export page = %q\n%s", p.Title, p.Markdown) + } + if p := byID["m-fm"]; p.Attrs["app"] != "com.apple.Notes" || !p.UpdatedAt.Equal(time.Date(2026, 11, 10, 16, 20, 0, 0, time.UTC)) { + t.Errorf("front-matter page = %v %v", p.Attrs, p.UpdatedAt) + } + if p := byID["c-1"]; p.Attrs["num_words"] != "13" || !strings.Contains(p.Markdown, "**App:** Slack") { + t.Errorf("csv page = %v\n%s", p.Attrs, p.Markdown) + } + if p := byID["j-1"]; !strings.Contains(p.Markdown, "**App:** Wispr Flow") { + t.Errorf("json page:\n%s", p.Markdown) + } +} + +func TestWisprExportDirPartial(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "bad.json"), []byte("{not json"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "good.ndjson"), []byte(`{"id":"g","text":"three words here","at":"2026-11-11T10:00:00Z"}`+"\n"), 0o600); err != nil { + t.Fatal(err) + } + c := newTestWispr(t, map[string]string{"path": dir}) + pages, cur, err := collectWispr(t, c, "") + if !IsPartial(err) || !strings.Contains(err.Error(), "bad.json") { + t.Fatalf("err = %v, want partial naming bad.json", err) + } + if len(pages) != 1 || pages[0].ID != "g" || cur != "2026-11-11T10:00:00Z" { + t.Fatalf("pages = %v cur = %q", wisprIDs(pages), cur) + } +} + +func TestWisprParseTime(t *testing.T) { + cases := map[string]time.Time{ + "2026-11-11 17:55:30.000 +00:00": time.Date(2026, 11, 11, 17, 55, 30, 0, time.UTC), + "2026-11-11 19:55:30.250 +02:00": time.Date(2026, 11, 11, 17, 55, 30, 250e6, time.UTC), + "2026-11-11T17:55:30Z": time.Date(2026, 11, 11, 17, 55, 30, 0, time.UTC), + "2026-11-11": time.Date(2026, 11, 11, 0, 0, 0, 0, time.UTC), + "garbage": {}, + } + for in, want := range cases { + if got := wisprParseTime(in); !got.Equal(want) { + t.Errorf("wisprParseTime(%q) = %v, want %v", in, got, want) + } + } + if got, ok := wisprTimeFromName("2026-11-11T14-05-09.md"); !ok || !got.Equal(time.Date(2026, 11, 11, 14, 5, 9, 0, time.UTC)) { + t.Errorf("wisprTimeFromName = %v %v", got, ok) + } + if _, ok := wisprTimeFromName("notes.md"); ok { + t.Error("wisprTimeFromName matched a name without a date") + } +} + +func TestWisprAppName(t *testing.T) { + for in, want := range map[string]string{"com.tinyspeck.slackmacgap": "Slack", "com.example.foo": "foo", "": "", "Cursor": "Cursor"} { + if got := wisprAppName(in); got != want { + t.Errorf("wisprAppName(%q) = %q, want %q", in, got, want) + } + } +} diff --git a/src/pkg/perspsrc/perspsrc.go b/src/pkg/perspsrc/perspsrc.go new file mode 100644 index 0000000000..0d03c4bc68 --- /dev/null +++ b/src/pkg/perspsrc/perspsrc.go @@ -0,0 +1,90 @@ +// Package perspsrc parses a live-linked review perspective set: a +// kind: ReviewPerspectives YAML document kept in a GitHub repo and named by +// review.perspectives_source (hivecommons/hive#17975), so maintainers can add, +// remove and re-describe perspectives through ordinary PRs. +// +// It is the review-set analogue of pkg/defsrc (which live-sources a whole +// agent definition) and follows the same design principles: +// +// - Validate before use: a document is only accepted when it would load as +// inline config would, via review.NewPerspectiveSet. A document that would +// not is an error, never a silent fall back to the built-in set. +// - Bounded input: a document over maxDocumentBytes is rejected outright. +// +// The document covers perspective names and focus text only. Cost and +// dispatch settings (combined_perspectives, plan_match, +// max_perspectives_per_pr) stay operator-owned in the hive config. +// +// This package, not pkg/config, owns the dependency on pkg/review, so +// pkg/config never has to import pkg/review. +package perspsrc + +import ( + "fmt" + "strings" + + "github.com/hivecommons/hive/pkg/review" + "gopkg.in/yaml.v3" +) + +// maxDocumentBytes caps how large a ReviewPerspectives document may be. A real +// one is a short list (well under a kilobyte per perspective); this bounds +// memory/log blast radius if a source path points at something huge. +const maxDocumentBytes = 512 * 1024 // 512 KiB + +// Kind is the required `kind` value of a review perspective document. +const Kind = "ReviewPerspectives" + +// Document is the YAML format of a live-linked review perspective set. +type Document struct { + Kind string `yaml:"kind" json:"kind"` + Perspectives []Entry `yaml:"perspectives" json:"perspectives"` +} + +// Entry is one perspective. Focus is optional for a built-in perspective +// (empty keeps the built-in text) and required for a hive-defined one. +type Entry struct { + Name string `yaml:"name" json:"name"` + Focus string `yaml:"focus,omitempty" json:"focus,omitempty"` +} + +// ParseDocument parses and validates a ReviewPerspectives document, returning +// the perspective names in dispatch order and the focus text keyed by name, in +// the same shape as review.perspectives and review.perspective_prompts. +func ParseDocument(doc string) (names []string, focus map[string]string, err error) { + if len(doc) > maxDocumentBytes { + return nil, nil, fmt.Errorf("perspectives document is %d bytes, over the limit of %d", len(doc), maxDocumentBytes) + } + var d Document + if err := yaml.Unmarshal([]byte(doc), &d); err != nil { + return nil, nil, fmt.Errorf("invalid perspectives YAML: %w", err) + } + if d.Kind != Kind { + return nil, nil, fmt.Errorf("expected kind %q, got %q", Kind, d.Kind) + } + if len(d.Perspectives) == 0 { + return nil, nil, fmt.Errorf("perspectives list is empty") + } + names = make([]string, 0, len(d.Perspectives)) + focus = map[string]string{} + seen := map[string]bool{} + for i, e := range d.Perspectives { + name := strings.TrimSpace(e.Name) + if name == "" { + return nil, nil, fmt.Errorf("perspectives[%d].name is required", i) + } + key := strings.ToLower(name) + if seen[key] { + return nil, nil, fmt.Errorf("duplicate perspective %q", name) + } + seen[key] = true + names = append(names, name) + if f := strings.TrimSpace(e.Focus); f != "" { + focus[name] = f + } + } + if _, err := review.NewPerspectiveSet(names, focus); err != nil { + return nil, nil, err + } + return names, focus, nil +} diff --git a/src/pkg/perspsrc/perspsrc_test.go b/src/pkg/perspsrc/perspsrc_test.go new file mode 100644 index 0000000000..ae352b834a --- /dev/null +++ b/src/pkg/perspsrc/perspsrc_test.go @@ -0,0 +1,116 @@ +package perspsrc + +import ( + "reflect" + "strings" + "testing" +) + +func TestParseDocument(t *testing.T) { + tests := []struct { + name string + doc string + wantNames []string + wantFocus map[string]string + wantErr string + }{ + { + name: "built-in only", + doc: `kind: ReviewPerspectives +perspectives: + - name: correctness + - name: security +`, + wantNames: []string{"correctness", "security"}, + wantFocus: map[string]string{}, + }, + { + name: "custom with focus", + doc: `kind: ReviewPerspectives +perspectives: + - name: correctness + - name: spyre-constraints + focus: "Head size a multiple of 64; fp16 only." +`, + wantNames: []string{"correctness", "spyre-constraints"}, + wantFocus: map[string]string{"spyre-constraints": "Head size a multiple of 64; fp16 only."}, + }, + { + name: "built-in with focus override", + doc: `kind: ReviewPerspectives +perspectives: + - name: style + focus: "Follow the repo's CONTRIBUTING.md conventions." +`, + wantNames: []string{"style"}, + wantFocus: map[string]string{"style": "Follow the repo's CONTRIBUTING.md conventions."}, + }, + { + name: "wrong kind", + doc: `kind: AgentDefinition +perspectives: + - name: correctness +`, + wantErr: `expected kind "ReviewPerspectives"`, + }, + { + name: "missing kind", + doc: `perspectives: + - name: correctness +`, + wantErr: `expected kind "ReviewPerspectives", got ""`, + }, + { + name: "empty list", + doc: "kind: ReviewPerspectives\nperspectives: []\n", + wantErr: "perspectives list is empty", + }, + { + name: "unknown name without focus", + doc: `kind: ReviewPerspectives +perspectives: + - name: sekurity +`, + wantErr: "has no focus text", + }, + { + name: "duplicate name", + doc: `kind: ReviewPerspectives +perspectives: + - name: security + - name: Security +`, + wantErr: "duplicate perspective", + }, + { + name: "oversize", + doc: "kind: ReviewPerspectives\n# " + strings.Repeat("x", maxDocumentBytes) + "\nperspectives:\n - name: correctness\n", + wantErr: "over the limit", + }, + { + name: "invalid yaml", + doc: "kind: [unterminated", + wantErr: "invalid perspectives YAML", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + names, focus, err := ParseDocument(tc.doc) + if tc.wantErr != "" { + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("ParseDocument() error = %v, want containing %q", err, tc.wantErr) + } + return + } + if err != nil { + t.Fatalf("ParseDocument() unexpected error: %v", err) + } + if !reflect.DeepEqual(names, tc.wantNames) { + t.Errorf("names = %v, want %v", names, tc.wantNames) + } + if !reflect.DeepEqual(focus, tc.wantFocus) { + t.Errorf("focus = %v, want %v", focus, tc.wantFocus) + } + }) + } +} diff --git a/src/pkg/snapshot/state.go b/src/pkg/snapshot/state.go index 935e5f9677..f3ceda0896 100644 --- a/src/pkg/snapshot/state.go +++ b/src/pkg/snapshot/state.go @@ -5,6 +5,7 @@ import ( "fmt" "log/slog" "os" + "path/filepath" "time" "github.com/hivecommons/hive/crux/watchdog" @@ -183,11 +184,26 @@ func SaveState(path string, state *PersistedState, logger *slog.Logger) error { return fmt.Errorf("marshaling state: %w", err) } - tmp := path + ".tmp" - if err := os.WriteFile(tmp, data, 0644); err != nil { + // A unique temp name per call: concurrent savers sharing path+".tmp" + // would rename each other's file away and fail with ENOENT (#17726). + f, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".*.tmp") + if err != nil { return fmt.Errorf("writing temp state file: %w", err) } + tmp := f.Name() + _, werr := f.Write(data) + if cerr := f.Close(); werr == nil { + werr = cerr + } + if werr == nil { + werr = os.Chmod(tmp, 0o644) + } + if werr != nil { + _ = os.Remove(tmp) + return fmt.Errorf("writing temp state file: %w", werr) + } if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) return fmt.Errorf("renaming state file: %w", err) } diff --git a/src/pkg/snapshot/state_test.go b/src/pkg/snapshot/state_test.go index 2d9905f7e2..61b6de9a78 100644 --- a/src/pkg/snapshot/state_test.go +++ b/src/pkg/snapshot/state_test.go @@ -6,6 +6,7 @@ import ( "os" "path/filepath" "strings" + "sync" "testing" "time" ) @@ -239,3 +240,31 @@ func TestLoadState_ReadErrorNotMissing(t *testing.T) { t.Error("state should be nil on read error") } } + +func TestSaveState_ConcurrentWritersDoNotRace(t *testing.T) { + path := filepath.Join(t.TempDir(), "state.json") + + var wg sync.WaitGroup + errs := make(chan error, 64) + for i := 0; i < 64; i++ { + wg.Add(1) + go func() { + defer wg.Done() + errs <- SaveState(path, &PersistedState{Agents: map[string]AgentState{}}, testLogger()) + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent SaveState: %v", err) + } + } + if _, err := LoadState(path, testLogger()); err != nil { + t.Fatalf("LoadState after concurrent saves: %v", err) + } + leftovers, _ := filepath.Glob(path + "*.tmp") + if len(leftovers) != 0 { + t.Fatalf("leftover temp files: %v", leftovers) + } +} diff --git a/src/pkg/tracing/reach.go b/src/pkg/tracing/reach.go index 78caf7461e..7bdb660b81 100644 --- a/src/pkg/tracing/reach.go +++ b/src/pkg/tracing/reach.go @@ -21,6 +21,7 @@ import ( "fmt" "log/slog" "os" + "path/filepath" "sort" "sync" "sync/atomic" @@ -388,11 +389,26 @@ func SaveReachState(path string) error { if err != nil { return fmt.Errorf("marshal reach state: %w", err) } - tmp := path + ".tmp" - if err := os.WriteFile(tmp, data, 0644); err != nil { + // Unique temp name per call so concurrent savers cannot rename each + // other's file away (ENOENT, #17728). + f, err := os.CreateTemp(filepath.Dir(path), filepath.Base(path)+".*.tmp") + if err != nil { return fmt.Errorf("write reach state: %w", err) } + tmp := f.Name() + _, werr := f.Write(data) + if cerr := f.Close(); werr == nil { + werr = cerr + } + if werr == nil { + werr = os.Chmod(tmp, 0o644) + } + if werr != nil { + _ = os.Remove(tmp) + return fmt.Errorf("write reach state: %w", werr) + } if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) return fmt.Errorf("rename reach state: %w", err) } return nil diff --git a/src/pkg/tracing/reach_test.go b/src/pkg/tracing/reach_test.go index bb5feb4a31..8a8da61d02 100644 --- a/src/pkg/tracing/reach_test.go +++ b/src/pkg/tracing/reach_test.go @@ -311,3 +311,31 @@ func TestReach_WindowRollover(t *testing.T) { c.winStart.Load(), c.winTotal.Load(), c.winErrors.Load()) } } + +func TestSaveReachState_ConcurrentWritersDoNotRace(t *testing.T) { + resetReach(t) + _, span := StartSpan(context.Background(), "governor.eval_cycle") + span.End() + path := filepath.Join(t.TempDir(), "reach-state.json") + + var wg sync.WaitGroup + errs := make(chan error, 64) + for i := 0; i < 64; i++ { + wg.Add(1) + go func() { + defer wg.Done() + errs <- SaveReachState(path) + }() + } + wg.Wait() + close(errs) + for err := range errs { + if err != nil { + t.Fatalf("concurrent SaveReachState: %v", err) + } + } + entries, _ := filepath.Glob(path + "*.tmp") + if len(entries) != 0 { + t.Fatalf("leftover temp files: %v", entries) + } +} diff --git a/src/scripts/check-action-pins.sh b/src/scripts/check-action-pins.sh index e60dd905a0..a143d5f9e3 100755 --- a/src/scripts/check-action-pins.sh +++ b/src/scripts/check-action-pins.sh @@ -17,6 +17,7 @@ set -uo pipefail DIR="${1:-.github/workflows}" fail=0 +RESOLVE_ERR="" # Every `uses:` on a third-party action must be a 40-hex SHA, not a tag or # branch. A moving ref is a supply-chain hole; that part is shape-only and needs @@ -34,6 +35,29 @@ while IFS= read -r line; do # DIR, so `${line%%:*}` names the workflow and not its line number. done < <(grep -rHnE '^\s*(-\s*)?uses:\s*[a-zA-Z0-9._-]+/' "$DIR" 2>/dev/null | grep -v '^\s*#') +# resolve_pin OWNER/REPO SHA — 0 the commit exists, 1 the API says it does not +# (422 "No commit found for SHA" or 404 for the repository itself), 2 the API +# gave no verdict. Only a definite 404/422 may be reported as MISSING-SHA: +# a rate limit, a 5xx or a dropped connection says nothing about the pin, and +# calling it "does not exist" sent crux main red on a README-only commit whose +# setup-go pin was tagged v7.0.0 upstream all along. Transient errors are +# retried briefly, then reported as UNRESOLVED so a re-run — not a pin bump — +# is the visible next step. +resolve_pin() { + local repo="$1" sha="$2" err attempt + for attempt in 1 2 3; do + if err="$(gh api "repos/$repo/commits/$sha" --jq '.sha' 2>&1 >/dev/null)"; then + return 0 + fi + case "$err" in + *"(HTTP 404)"*|*"(HTTP 422)"*) return 1 ;; + esac + [ "$attempt" -lt 3 ] && sleep "$attempt" + done + RESOLVE_ERR="$(printf '%s' "$err" | head -1)" + return 2 +} + # Resolve each unique pin. Skipped without a token so the script stays usable # offline; CI always has one. if command -v gh >/dev/null 2>&1; then @@ -51,10 +75,17 @@ if command -v gh >/dev/null 2>&1; then [ -z "$ref" ] && continue sha="${ref#*@}"; path="${ref%@*}" repo="$(printf '%s' "$path" | cut -d/ -f1,2)" - if ! gh api "repos/$repo/commits/$sha" --jq '.sha' >/dev/null 2>&1; then - echo "MISSING-SHA $repo@$sha does not exist in that repository" - fail=1 - fi + resolve_pin "$repo" "$sha" + case "$?" in + 1) + echo "MISSING-SHA $repo@$sha does not exist in that repository" + fail=1 + ;; + 2) + echo "UNRESOLVED $repo@$sha: the GitHub API gave no verdict after 3 attempts ($RESOLVE_ERR); re-run, this is not a missing pin" + fail=1 + ;; + esac done <<<"$pins" fi diff --git a/src/scripts/feedback-gate.sh b/src/scripts/feedback-gate.sh new file mode 100755 index 0000000000..50c1c0e314 --- /dev/null +++ b/src/scripts/feedback-gate.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# feedback-gate.sh — decides whether the post-merge "Thank you for your +# contribution!" comment (.github/workflows/feedback.yml → +# hivecommons/infra reusable-feedback.yml) should be posted on a merged PR. +# +# The reusable workflow is unconditional, so before #17863 every merged PR +# got the comment: 38 of 40 in the 2026-11-11 spam sample were on hive-App or +# maintainer PRs that nobody reads it on. The comment is for people merging +# their first change, so the gate keeps it for exactly that case. +# +# Reads the pull_request event from $GITHUB_EVENT_PATH (or the FEEDBACK_PR_* +# overrides below, used by the self-test) and writes `post=true|false` and +# `reason=<why>` to $GITHUB_OUTPUT (stdout when unset). Exit status is 0 in +# both cases; only a broken event or API read exits non-zero, which the caller +# treats as "do not post". +# +# Configuration (workflow `env:`), each with its default: +# FEEDBACK_SKIP_BOTS=1 skip `[bot]` / type=Bot authors +# FEEDBACK_SKIP_BRANCHES='scanner/* hive/*' +# space-separated globs of hive-managed +# head branches to skip +# FEEDBACK_SKIP_HIVE_ORIGIN=1 skip PRs whose body carries a `<!-- hive:` +# marker (hive-agent PRs on any branch, +# e.g. `fix-*`) +# FEEDBACK_SKIP_LOGINS='' space-separated logins never thanked +# FEEDBACK_FIRST_TIME_ONLY=1 skip authors with another merged PR in +# the repo (needs GH_TOKEN; a failed lookup +# skips the comment rather than guessing) +# +# Overrides for tests / manual runs: FEEDBACK_PR_AUTHOR, FEEDBACK_PR_AUTHOR_TYPE, +# FEEDBACK_PR_HEAD_REF, FEEDBACK_PR_BODY, FEEDBACK_PR_MERGED, FEEDBACK_PR_NUMBER, +# FEEDBACK_MERGED_COUNT (skips the API lookup). +set -euo pipefail + +skip_bots="${FEEDBACK_SKIP_BOTS:-1}" +skip_branches="${FEEDBACK_SKIP_BRANCHES-scanner/* hive/*}" +skip_hive_origin="${FEEDBACK_SKIP_HIVE_ORIGIN:-1}" +skip_logins="${FEEDBACK_SKIP_LOGINS:-}" +first_time_only="${FEEDBACK_FIRST_TIME_ONLY:-1}" +repo="${GITHUB_REPOSITORY:-}" + +lower() { printf '%s' "$1" | tr '[:upper:]' '[:lower:]'; } + +event_field() { + local path=$1 + if [[ -n "${GITHUB_EVENT_PATH:-}" && -f "${GITHUB_EVENT_PATH}" ]]; then + jq -r "${path} // empty" "$GITHUB_EVENT_PATH" + fi +} + +author="${FEEDBACK_PR_AUTHOR:-$(event_field '.pull_request.user.login')}" +author_type="${FEEDBACK_PR_AUTHOR_TYPE:-$(event_field '.pull_request.user.type')}" +head_ref="${FEEDBACK_PR_HEAD_REF:-$(event_field '.pull_request.head.ref')}" +body="${FEEDBACK_PR_BODY-$(event_field '.pull_request.body')}" +merged="${FEEDBACK_PR_MERGED:-$(event_field '.pull_request.merged')}" +number="${FEEDBACK_PR_NUMBER:-$(event_field '.pull_request.number')}" + +decide() { + local post=$1 reason=$2 + if [[ -n "${GITHUB_OUTPUT:-}" ]]; then + printf 'post=%s\nreason=%s\n' "$post" "$reason" >> "$GITHUB_OUTPUT" + fi + printf 'feedback-gate: post=%s (%s)\n' "$post" "$reason" + exit 0 +} + +[[ -n "$author" ]] || { echo 'feedback-gate: no PR author in the event' >&2; exit 1; } +[[ "$merged" == 'true' ]] || decide false 'pr-not-merged' + +if [[ "$skip_bots" == '1' ]] && { [[ "$author" == *'[bot]' ]] || [[ "$(lower "$author_type")" == 'bot' ]]; }; then + decide false "bot-author:${author}" +fi + +for login in $skip_logins; do + if [[ "$(lower "$login")" == "$(lower "$author")" ]]; then + decide false "listed-login:${author}" + fi +done + +for pattern in $skip_branches; do + # shellcheck disable=SC2053 # the glob in $pattern is meant to expand + if [[ -n "$head_ref" && "$head_ref" == $pattern ]]; then + decide false "hive-branch:${head_ref}" + fi +done + +if [[ "$skip_hive_origin" == '1' && "$body" == *'<!-- hive:'* ]]; then + decide false 'hive-origin-body' +fi + +if [[ "$first_time_only" == '1' ]]; then + merged_count="${FEEDBACK_MERGED_COUNT:-}" + if [[ -z "$merged_count" ]]; then + [[ -n "$repo" ]] || { echo 'feedback-gate: GITHUB_REPOSITORY unset' >&2; exit 1; } + # The event fires after the merge, so the PR being judged is already one + # of the author's merged PRs: "first-time" means it is the only one. + merged_count=$(gh api -X GET search/issues -f "q=repo:${repo} is:pr is:merged author:${author}" -F per_page=1 --jq '.total_count') \ + || { echo "feedback-gate: merged-PR lookup for ${author} failed; not posting" >&2; exit 1; } + fi + if [[ ! "$merged_count" =~ ^[0-9]+$ ]]; then + echo "feedback-gate: unexpected merged count '${merged_count}'" >&2 + exit 1 + fi + if (( merged_count > 1 )); then + decide false "returning-contributor:${author}:${merged_count}-merged" + fi +fi + +decide true "first-merged-pr:${author}${number:+:#${number}}" diff --git a/src/scripts/mutation-report.sh b/src/scripts/mutation-report.sh index 215e22836f..49a9d09eab 100755 --- a/src/scripts/mutation-report.sh +++ b/src/scripts/mutation-report.sh @@ -100,11 +100,33 @@ post_comment() { echo "PR_NUMBER/GITHUB_REPOSITORY unset; not posting a comment." return 0 fi + # The report always lands in the step summary; the PR comment is reserved + # for findings (any verdict but `pass`). A clean run never creates a + # comment, and one that follows an earlier finding edits that comment down + # to a one-line resolved note instead of posting again (#17863). + if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then + cat "$body" >> "$GITHUB_STEP_SUMMARY" + fi + local verdict="" resolved="" + [ -f "$verdict_file" ] && verdict=$(cat "$verdict_file") + if [ "$verdict" = pass ]; then + # shellcheck disable=SC2016 # backticks are literal Markdown + resolved="$(printf '%s\n### Mutation testing\n\n✅ Resolved: no surviving or uncovered mutants in changed files on the current head.\n\n_Verdict: `pass` (#17349)._\n' "$marker")" + fi # Only upsert the bot's own report so another participant cannot pre-seat or # get overwritten by a comment that merely starts with the marker. local id id=$(gh api --paginate "repos/${repo}/issues/${pr_number}/comments" \ --jq ".[] | select(.user.login == \"github-actions[bot]\" and (.body | startswith(\"${marker}\"))) | .id" 2>/dev/null | head -n 1 || true) + if [ -n "$resolved" ]; then + if [ -z "$id" ]; then + echo "mutation: verdict pass and no earlier comment; not posting (report is in the step summary)." + return 0 + fi + gh api -X PATCH "repos/${repo}/issues/comments/${id}" -f "body=${resolved}" >/dev/null \ + || echo "::warning::could not edit mutation comment" + return 0 + fi if [ -n "$id" ]; then gh api -X PATCH "repos/${repo}/issues/comments/${id}" -F "body=@${body}" >/dev/null \ || echo "::warning::could not edit mutation comment" diff --git a/src/scripts/test-check-action-pins.sh b/src/scripts/test-check-action-pins.sh index c796f1384b..0d70f4fcaa 100755 --- a/src/scripts/test-check-action-pins.sh +++ b/src/scripts/test-check-action-pins.sh @@ -48,14 +48,19 @@ assert_eq() { # Fake `gh`: only `gh api repos/<owner>/<repo>/commits/<sha> --jq .sha` is # ever called by the check. Every call is appended to $FAKE_GH_LOG; a call # whose SHA equals $FAKE_GH_MISSING fails like the real API does for a commit -# that does not exist in the repository. +# that does not exist in the repository (HTTP 422 on stderr), and one whose +# SHA equals $FAKE_GH_TRANSIENT fails like a rate limit (HTTP 403), which is +# not a verdict on the pin. BIN="$TMP/bin" mkdir -p "$BIN" cat >"$BIN/gh" <<'SH' #!/usr/bin/env bash printf '%s\n' "$*" >>"${FAKE_GH_LOG:?}" case "$2" in - *"/commits/${FAKE_GH_MISSING:-no-such-sha}") exit 1 ;; + *"/commits/${FAKE_GH_MISSING:-no-such-sha}") + echo "gh: No commit found for SHA: ${2##*/} (HTTP 422)" >&2; exit 1 ;; + *"/commits/${FAKE_GH_TRANSIENT:-no-such-sha}") + echo "gh: API rate limit exceeded for installation ID 1 (HTTP 403)" >&2; exit 1 ;; esac printf '%s\n' "${2##*/}" exit 0 @@ -183,7 +188,8 @@ cat >"$BIN/gh" <<SH #!/usr/bin/env bash printf '%s\n' "\$*" >>"\${FAKE_GH_LOG:?}" case "\$2" in - */commits/\${FAKE_GH_MISSING:-no-such-sha}|*/commits/$FAKE_SECOND_MISSING) exit 1 ;; + */commits/\${FAKE_GH_MISSING:-no-such-sha}|*/commits/$FAKE_SECOND_MISSING) + echo "gh: No commit found for SHA: \${2##*/} (HTTP 422)" >&2; exit 1 ;; esac printf '%s\n' "\${2##*/}" exit 0 @@ -199,12 +205,33 @@ cat >"$BIN/gh" <<'SH' #!/usr/bin/env bash printf '%s\n' "$*" >>"${FAKE_GH_LOG:?}" case "$2" in - *"/commits/${FAKE_GH_MISSING:-no-such-sha}") exit 1 ;; + *"/commits/${FAKE_GH_MISSING:-no-such-sha}") + echo "gh: No commit found for SHA: ${2##*/} (HTTP 422)" >&2; exit 1 ;; + *"/commits/${FAKE_GH_TRANSIENT:-no-such-sha}") + echo "gh: API rate limit exceeded for installation ID 1 (HTTP 403)" >&2; exit 1 ;; esac printf '%s\n' "${2##*/}" exit 0 SH +echo "case: an API error that is not 404/422 is not a missing SHA" +D="$TMP/transient" +workflow "$D" ci.yml <<EOF +jobs: + build: + steps: + - uses: actions/checkout@$SHA_A + - uses: actions/setup-go@$SHA_B # v7.0.0 +EOF +run_check transient FAKE_GH_TRANSIENT="$SHA_B" -- "$D" +assert_exit 1 "$RC" "an unresolved pin still fails closed" +assert_not_contains "$OUT" "MISSING-SHA" "a rate limit is not reported as a missing SHA" +assert_contains "$OUT" "UNRESOLVED actions/setup-go@$SHA_B" "reports the pin as UNRESOLVED instead" +assert_contains "$OUT" "(HTTP 403)" "quotes the API error" +assert_contains "$OUT" "action pin check FAILED" "prints the failure banner" +assert_eq 3 "$(grep -c "commits/$SHA_B" <<<"$GH_CALLS")" "the transient failure is retried three times" +assert_eq 1 "$(grep -c "commits/$SHA_A" <<<"$GH_CALLS")" "a pin that resolves first time is not retried" + echo "case: a directory with no third-party actions passes with gh present" D="$TMP/nopins" workflow "$D" ci.yml <<EOF diff --git a/src/scripts/test-feedback-gate.sh b/src/scripts/test-feedback-gate.sh new file mode 100755 index 0000000000..3864f8ff04 --- /dev/null +++ b/src/scripts/test-feedback-gate.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# test-feedback-gate.sh — pins the gating rules of feedback-gate.sh (#17863) +# with a stub `gh` so no network access is needed. +set -u -o pipefail + +HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPT="${HERE}/feedback-gate.sh" +TMP_ROOT="${HERE}/../.test-tmp" +mkdir -p "$TMP_ROOT" +TMP="$(mktemp -d "$TMP_ROOT/feedback-gate.XXXXXX")" +trap 'rm -rf "$TMP"' EXIT + +fail=0 +pass() { echo " ok: $*"; } +bad() { echo " FAIL: $*"; fail=1; } + +mkdir -p "$TMP/bin" +cat > "$TMP/bin/gh" <<'GHSTUB' +#!/usr/bin/env bash +case "$*" in + *search/issues*) + [ -n "${GH_FAIL:-}" ] && exit 1 + printf '%s\n' "${GH_TOTAL:-1}" ;; + *) echo "unexpected gh call: $*" >&2; exit 2 ;; +esac +GHSTUB +chmod +x "$TMP/bin/gh" + +# run <expected-post> <label> [VAR=value ...] +run() { + local want=$1 label=$2; shift 2 + local out="$TMP/output"; : > "$out" + local result rc + result=$(env -i PATH="$TMP/bin:/usr/bin:/bin" HOME="$TMP" GITHUB_OUTPUT="$out" GITHUB_REPOSITORY=example/hive \ + FEEDBACK_PR_MERGED=true FEEDBACK_PR_AUTHOR=newcomer FEEDBACK_PR_AUTHOR_TYPE=User \ + FEEDBACK_PR_HEAD_REF=feat/thing FEEDBACK_PR_BODY='plain body' FEEDBACK_PR_NUMBER=7 \ + "$@" bash "$SCRIPT" 2>&1); rc=$? + if [ "$want" = error ]; then + [ "$rc" -ne 0 ] && ! grep -q '^post=' "$out" && pass "$label (exit $rc, no output)" || bad "$label: rc=$rc out=$(cat "$out") $result" + return + fi + if [ "$rc" -eq 0 ] && grep -qx "post=$want" "$out"; then pass "$label ($(grep '^reason=' "$out"))"; else bad "$label: rc=$rc out=$(cat "$out") $result"; fi +} + +echo "default rules" +run true 'human first merged PR is thanked' GH_TOTAL=1 +run false 'returning contributor is not thanked' GH_TOTAL=3 +run false 'App author is skipped' FEEDBACK_PR_AUTHOR='hivecommons-hive[bot]' FEEDBACK_PR_AUTHOR_TYPE=Bot +run false 'type=Bot without [bot] suffix is skipped' FEEDBACK_PR_AUTHOR=somebot FEEDBACK_PR_AUTHOR_TYPE=Bot +run false 'scanner/* branch is skipped' FEEDBACK_PR_HEAD_REF=scanner/v6-claude-auth +run false 'hive/* branch is skipped' FEEDBACK_PR_HEAD_REF=hive/adopt-123 +run false 'hive-origin body is skipped on any branch' FEEDBACK_PR_HEAD_REF=fix-17863-feedback FEEDBACK_PR_BODY=$'Fixes #1\n<!-- hive:origin issue=1 -->' +run true 'fix-* branch by a person is still thanked' FEEDBACK_PR_HEAD_REF=fix-typo GH_TOTAL=1 +run false 'unmerged (closed) PR is never thanked' FEEDBACK_PR_MERGED=false +run error 'merged-PR lookup failure exits non-zero without an output' GH_FAIL=1 +run error 'garbage merged count exits non-zero' GH_TOTAL=lots + +echo "configuration" +run true 'FEEDBACK_SKIP_BOTS=0 thanks a bot' FEEDBACK_SKIP_BOTS=0 FEEDBACK_PR_AUTHOR='x[bot]' FEEDBACK_PR_AUTHOR_TYPE=Bot GH_TOTAL=1 +run true 'FEEDBACK_FIRST_TIME_ONLY=0 thanks a returning contributor' FEEDBACK_FIRST_TIME_ONLY=0 GH_TOTAL=9 +run false 'FEEDBACK_SKIP_BRANCHES adds a pattern' FEEDBACK_SKIP_BRANCHES='scanner/* release/*' FEEDBACK_PR_HEAD_REF=release/v5.1 +run true 'FEEDBACK_SKIP_BRANCHES= disables branch skipping' FEEDBACK_SKIP_BRANCHES= FEEDBACK_PR_HEAD_REF=scanner/x GH_TOTAL=1 +run false 'FEEDBACK_SKIP_LOGINS is case-insensitive' FEEDBACK_SKIP_LOGINS='Maintainer other' FEEDBACK_PR_AUTHOR=maintainer +run true 'FEEDBACK_MERGED_COUNT override skips the API' FEEDBACK_MERGED_COUNT=1 GH_FAIL=1 + +echo "event file" +cat > "$TMP/event.json" <<'EOF' +{"pull_request":{"number":17716,"merged":true,"user":{"login":"clubanderson","type":"User"},"head":{"ref":"fix/thing"},"body":"body"}} +EOF +out="$TMP/output"; : > "$out" +result=$(env -i PATH="$TMP/bin:/usr/bin:/bin" HOME="$TMP" GITHUB_OUTPUT="$out" GITHUB_REPOSITORY=example/hive \ + GITHUB_EVENT_PATH="$TMP/event.json" GH_TOTAL=400 bash "$SCRIPT" 2>&1); rc=$? +[ "$rc" -eq 0 ] && grep -qx 'post=false' "$out" && grep -q 'returning-contributor:clubanderson:400-merged' "$out" \ + && pass "fields are read from GITHUB_EVENT_PATH" || bad "event file: rc=$rc out=$(cat "$out") $result" + +if [ "$fail" -ne 0 ]; then + echo "test-feedback-gate.sh: FAILED" + exit 1 +fi +echo "test-feedback-gate.sh: all checks passed" diff --git a/src/scripts/test-mutation-report.sh b/src/scripts/test-mutation-report.sh index bb10e92ec6..02b5f1e801 100755 --- a/src/scripts/test-mutation-report.sh +++ b/src/scripts/test-mutation-report.sh @@ -203,6 +203,43 @@ output=$(MUTATION_OUT_DIR="$TMP/missing" bash "$SCRIPT" verdict 2>&1); rc=$? [ "$rc" -eq 1 ] && out_grep -q '::error::mutation: no verdict at' \ && pass "missing verdict file fails the verdict phase" || bad "missing verdict rc=$rc: $output" +echo "post phase (#17863)" +# Stub gh: lists $GH_EXISTING_ID (if set) as the bot's marker comment and logs writes. +cat > "$TMP/bin/gh" <<'GHSTUB' +#!/usr/bin/env bash +case "$*" in + *-X*) printf '%s\n' "$*" >> "$GH_WRITE_LOG"; exit 0 ;; + *'/comments'*) if [ -n "${GH_EXISTING_ID:-}" ]; then printf '%s\n' "$GH_EXISTING_ID"; fi; exit 0 ;; + *) echo "unexpected gh call: $*" >&2; exit 2 ;; +esac +GHSTUB +chmod +x "$TMP/bin/gh" +run_post() { # run_post <out-dir> [existing-id] + GH_WRITE_LOG="$TMP/gh-writes.log"; : > "$GH_WRITE_LOG" + SUMMARY="$TMP/summary.md"; : > "$SUMMARY" + output=$(cd "$repo" && GH_WRITE_LOG="$GH_WRITE_LOG" GH_EXISTING_ID="${2:-}" GITHUB_STEP_SUMMARY="$SUMMARY" \ + MUTATION_OUT_DIR="$1" PR_NUMBER=42 GITHUB_REPOSITORY=example/hive bash "$SCRIPT" post 2>&1) + writes=$(cat "$GH_WRITE_LOG") +} +mkdir -p "$TMP/out-pass" "$TMP/out-lived" +printf '<!-- hive-mutation -->\n### Mutation testing\n\nAll 3 mutant(s) in changed files were killed.\n' > "$TMP/out-pass/comment.md" +echo pass > "$TMP/out-pass/verdict" +printf '<!-- hive-mutation -->\n### Mutation testing\n\ngremlins found **1 surviving mutant(s)**\n' > "$TMP/out-lived/comment.md" +echo lived > "$TMP/out-lived/verdict" +run_post "$TMP/out-pass" +[ -z "$writes" ] && pass "pass verdict with no earlier comment posts nothing" || bad "pass posted: $writes" +out_grep -q 'verdict pass and no earlier comment; not posting' && pass "skip is logged" || bad "skip not logged: $output" +grep -q 'All 3 mutant(s) in changed files were killed' "$SUMMARY" && pass "clean report still lands in the step summary" || bad "summary missing report" +run_post "$TMP/out-lived" +grep -q "^api -X POST repos/example/hive/issues/42/comments" <<<"$writes" && pass "lived verdict posts a new comment" || bad "lived did not post: $writes" +run_post "$TMP/out-lived" 99 +grep -q "^api -X PATCH repos/example/hive/issues/comments/99" <<<"$writes" && pass "lived verdict edits the existing comment" || bad "lived did not edit: $writes" +grep -q 'POST' <<<"$writes" && bad "lived posted a duplicate" || pass "no duplicate comment" +run_post "$TMP/out-pass" 99 +grep -q "^api -X PATCH repos/example/hive/issues/comments/99 -f body=<!-- hive-mutation -->" <<<"$writes" && pass "pass after a finding edits the existing comment, keeping the marker" || bad "pass did not edit: $writes" +grep -q 'Resolved: no surviving or uncovered mutants' <<<"$writes" && pass "edited body is the one-line resolved note" || bad "resolved note missing: $writes" +grep -q 'All 3 mutant' <<<"$writes" && bad "full report was written to the comment" || pass "full report stays out of the comment" + if [ "$fail" -ne 0 ]; then echo "test-mutation-report.sh: FAILED" exit 1