diff --git a/.github/scripts/test-v6-topup-autoresolve.sh b/.github/scripts/test-v6-topup-autoresolve.sh index 46ea8ae185..1e37b1c954 100755 --- a/.github/scripts/test-v6-topup-autoresolve.sh +++ b/.github/scripts/test-v6-topup-autoresolve.sh @@ -63,7 +63,6 @@ case_all_known() { echo v5bot > discord/bot.go printf '# Changelog\n\n## Unreleased\n\n## 1.1.0 (v5)\n\n- v5 entry\n\n## 1.0.0\n\n- base\n' > CHANGELOG.md echo v5 > src/scripts/b.sh - echo 'package config // v5 monolith edit' > src/pkg/config/config.go echo A5 > src/pkg/dashboard/static-src/a.html printf 'A5\nB\n' > src/pkg/dashboard/static/index.html git add -A && git commit -qm v5 && finish_pin v5 @@ -79,7 +78,7 @@ case_all_known() { bash "$RESOLVER" git commit -q --no-edit [ ! -e discord/bot.go ] || { echo "discord not deleted"; return 1; } - grep -q 'v6 slim' src/pkg/config/config.go || { echo "config.go not v6"; return 1; } + grep -q 'v6 slim' src/pkg/config/config.go || { echo "config.go lost v6's one-sided edit"; return 1; } grep -q '6.1.0 (v6)' CHANGELOG.md && grep -q '1.1.0 (v5)' CHANGELOG.md || { echo "changelog lost a side"; return 1; } ! grep -qE '^(<<<<<<<|=======|>>>>>>>)' CHANGELOG.md .github/scripts/integration-governance.cjs || { echo "markers left"; return 1; } [ "$(cat src/pkg/dashboard/static/index.html)" = "$(printf 'A5\nB6')" ] || { echo "index not regenerated"; return 1; } @@ -106,7 +105,33 @@ case_pin_file_real_edit() { bash "$RESOLVER" } +# v5 adds a real field to the monolith config.go while v6 has split it. The +# resolver must refuse (keeping v6's side would drop the field silently) and +# must leave config.go unmerged for the human, even when every other conflict +# is a known pattern. +case_config_real_field() { + new_repo "$WORK/config" && cd "$WORK/config" + printf 'package config // base\n\ntype Config struct {\n\tMeetings MeetingsConfig `yaml:"meetings"`\n}\n' > src/pkg/config/config.go + printf '# Changelog\n\n## Unreleased\n\n## 1.1.0 (v5)\n\n- v5 entry\n\n## 1.0.0\n\n- base\n' > CHANGELOG.md + git commit -qam "v5 adds a config field" + git checkout -q v6 + echo 'package config // v6 slim (structs moved to types.go)' > src/pkg/config/config.go + printf '# Changelog\n\n## Unreleased\n\n## 6.1.0 (v6)\n\n- v6 entry\n\n## 1.0.0\n\n- base\n' > CHANGELOG.md + git commit -qam "v6 splits config.go" + git merge --no-ff -q v5 -m merge >/dev/null 2>&1 && { echo "expected conflicts"; return 1; } + # Exit 0 here means the resolver accepted the conflict, which is the bug; + # `expect ... fail` turns that into a test failure. + local out rc=0 + out="$(bash "$RESOLVER" 2>&1)" || rc=$? + printf '%s\n' "$out" + [ "$rc" -ne 0 ] || return 0 + git status --porcelain | grep -q '^UU src/pkg/config/config.go$' || { echo "config.go was resolved"; return 0; } + printf '%s\n' "$out" | grep -q 'port v5.s change by hand' || { echo "no hand-port hint"; return 0; } + return 1 +} + expect all-known-patterns-resolve ok case_all_known +expect config-go-real-field-refused fail case_config_real_field expect unknown-conflict-refused fail case_unknown_conflict expect pin-file-real-edit-refused fail case_pin_file_real_edit diff --git a/.github/scripts/v6-topup-autoresolve.sh b/.github/scripts/v6-topup-autoresolve.sh index faa9333b54..252ddb5f45 100755 --- a/.github/scripts/v6-topup-autoresolve.sh +++ b/.github/scripts/v6-topup-autoresolve.sh @@ -10,8 +10,6 @@ # Known patterns: # - DU discord/** and DU/UD .github/workflows/discord-syntax.yml: v6 removed # discord/ (#9277) while v5 still develops it. Keep v6's deletion. -# - UU src/pkg/config/config.go: v6 split the monolith into topical files -# (#9386, #9956). Keep v6's side. # - UU CHANGELOG.md: both lines prepend their own release sections under # "## Unreleased" on every release, so the top of the file conflicts on # almost every top-up. Keep both sides' lines (v6's block first, then @@ -25,6 +23,17 @@ # it as the byte-exact concatenation of static-src/manifest.txt (what # webstatic.ConcatIndex and TestIndexHTMLIsGenerated do), provided no # static-src file is itself conflicted. +# +# Deliberately NOT a known pattern: +# - UU src/pkg/config/config.go. v6 split the monolith into topical files +# (#9386, #9956), and an earlier version of this script kept v6's side of +# any config.go conflict. A conflict there means v5 changed config.go since +# the last top-up, and keeping v6's side drops that change without a trace: +# #18222's keystore sealing survived only because it was hand-ported into +# v6's load.go/save.go. A config.go conflict always goes to a human, who +# ports v5's change into the matching split file. Whether v6 already has an +# equivalent change cannot be decided mechanically (the code moved), so +# there is no "safe subset" to auto-resolve. set -euo pipefail PIN_FILE=.github/scripts/integration-governance.cjs @@ -35,10 +44,14 @@ PIN_RE="^const TRUSTED_SCRIPTS_TREE = '[0-9a-f]{40}';\$" unmerged="$(git status --porcelain | grep -E '^(DD|AU|UD|UA|DU|AA|UU) ' || true)" [ -n "$unmerged" ] || { echo "v6-topup-autoresolve: no conflicts to resolve"; exit 1; } -known='^DU discord/|^(DU|UD) \.github/workflows/discord-syntax\.yml$|^UU src/pkg/config/config\.go$|^UU CHANGELOG\.md$|^UU \.github/scripts/integration-governance\.cjs$|^UU src/pkg/dashboard/static/index\.html$' +known='^DU discord/|^(DU|UD) \.github/workflows/discord-syntax\.yml$|^UU CHANGELOG\.md$|^UU \.github/scripts/integration-governance\.cjs$|^UU src/pkg/dashboard/static/index\.html$' if unknown="$(printf '%s\n' "$unmerged" | grep -vE "$known")"; then echo "v6-topup-autoresolve: conflicts outside the known patterns:" printf '%s\n' "$unknown" + if printf '%s\n' "$unknown" | grep -q ' src/pkg/config/config\.go$'; then + echo "v6-topup-autoresolve: v5 changed src/pkg/config/config.go, which v6 split into topical files;" + echo " port v5's change by hand: git diff \$(git merge-base HEAD MERGE_HEAD) MERGE_HEAD -- src/pkg/config/config.go" + fi exit 1 fi @@ -70,10 +83,6 @@ while IFS= read -r entry; do "DU "*|"UD "*) git rm -q -- "$file" ;; - "UU src/pkg/config/config.go") - git checkout --ours -- "$file" - git add -- "$file" - ;; "UU CHANGELOG.md") tmp="$(mktemp -d)" git show ":1:$file" > "$tmp/base" @@ -107,7 +116,11 @@ if [ "$regen_index" = 1 ]; then while IFS= read -r line || [ -n "$line" ]; do line="$(printf '%s' "$line" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')" case "$line" in ''|'#'*) continue ;; esac - case "$line" in /*|..*) echo "v6-topup-autoresolve: unsafe manifest path $line"; exit 1 ;; esac + # The same clean-relative-path rule as webstatic.ParseIndexManifest. + case "$line" in + /*|..*|./*|*/|*//*|*/./*|*/../*|*/.|*/..) + echo "v6-topup-autoresolve: unsafe manifest path $line"; exit 1 ;; + esac cat "$STATIC_SRC/$line" >> "$INDEX_FILE" done < "$STATIC_SRC/manifest.txt" git add -- "$INDEX_FILE" diff --git a/.github/workflows/v6-topup.yml b/.github/workflows/v6-topup.yml index dd58072398..c8142c43d7 100644 --- a/.github/workflows/v6-topup.yml +++ b/.github/workflows/v6-topup.yml @@ -93,7 +93,6 @@ jobs: # them with their history): # - discord/ + discord-syntax.yml modify/delete: keep v6's # deletion (#9277, #11551). - # - src/pkg/config/config.go: keep v6's split (#9386, #9956). # - CHANGELOG.md: both lines prepend release sections, keep both. # - integration-governance.cjs TRUSTED_SCRIPTS_TREE literal only: # re-pin to the merged src/scripts tree. @@ -101,6 +100,8 @@ jobs: # When the ONLY conflicts match those patterns, resolve them and # finish the merge. Anything else (other content conflicts, other # deletions, other hunks in the pin file) still fails loudly below. + # src/pkg/config/config.go is deliberately NOT auto-resolved: v6 + # split it, so keeping v6's side would silently drop v5's change. # The resolver runs from the copy taken off the triggering ref # before the checkout, not from v6's tree. elif bash "$RUNNER_TEMP/v6-topup-autoresolve.sh"; then diff --git a/CHANGELOG.md b/CHANGELOG.md index b554d6764e..1db025f6f1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -93,6 +93,15 @@ Hive did not historically maintain a complete changelog. This file starts a prag - The signed-commit reconciler's intentional refusal to rewrite a foreign or merge commit no longer files repeated unclassified-error fleet reports; its warning and PR remediation remain visible ([#17924](https://github.com/hivecommons/hive/issues/17924)). - Dashboard logs one summary warning per five minutes for dropped SSE events instead of one warning per event, so a stalled browser tab no longer triggers repeated-error fleet reports (#17995). + +## 2026-11-15 (v5.269.0) + +### Added + +- Crux gains ID-addressed, queued kicks: `Manager.EnqueueKick(agent, kickID, message)` appends a kick to a bounded per-agent FIFO (`HIVE_KICK_QUEUE_SIZE`, default 8) and returns its kick ID (caller-supplied or generated), delivering each kick once the agent is idle. `Manager.KickStatus(kickID)` reports `queued`, `pending`, `delivered`, `completed` (with the turn's outcome) or `failed`; waits and turns are bounded by `HIVE_KICK_QUEUE_WAIT_TIMEOUT` (30m) and `HIVE_KICK_QUEUE_TURN_TIMEOUT` (60m). The optional `crux.KickQueuer` interface exposes it; `SendKick` and `SendKickAsync` are unchanged. Groundwork for meeting research kicks ([#18285](https://github.com/hivecommons/hive/issues/18285), parent [#18284](https://github.com/hivecommons/hive/issues/18284)). +- Crux kick observers fan out: `Manager.AddKickObserver(func(crux.KickEvent)) (remove func())` registers any number of observers, each receiving every kick event in order on its own goroutine (a slow observer has events dropped rather than stalling the runtime). Events carry the kick ID of the queued kick they belong to, and the kick queue adds `kick-queued`, `kick-completed` and `kick-failed`. `SetKickObserver` keeps its single slot and its two events, so the Linear agent behaves exactly as before ([#18286](https://github.com/hivecommons/hive/issues/18286), parent [#18284](https://github.com/hivecommons/hive/issues/18284)). +- Inbox: Google Meet meetings as a knowledge integration — a `google-meet` connector (user-level OAuth with the hosted paste flow, or a service account with domain-wide delegation) that syncs conference records with a recording, transcript or Gemini notes into one fact each, with the event title, participants, Gemini notes as the summary and the transcript entries as `[mm:ss] Speaker: text` lines; a default `inbox.sources` entry with a Connect / Check / Sync now card (Meet recordings are Drive files behind a bearer token, so Inbox playback links the Drive page). + ## 2026-11-15 (v5.268.0) ### Added diff --git a/changelog.d/added-18285-kick-queue.md b/changelog.d/added-18285-kick-queue.md deleted file mode 100644 index fce39931f9..0000000000 --- a/changelog.d/added-18285-kick-queue.md +++ /dev/null @@ -1 +0,0 @@ -- Crux gains ID-addressed, queued kicks: `Manager.EnqueueKick(agent, kickID, message)` appends a kick to a bounded per-agent FIFO (`HIVE_KICK_QUEUE_SIZE`, default 8) and returns its kick ID (caller-supplied or generated), delivering each kick once the agent is idle. `Manager.KickStatus(kickID)` reports `queued`, `pending`, `delivered`, `completed` (with the turn's outcome) or `failed`; waits and turns are bounded by `HIVE_KICK_QUEUE_WAIT_TIMEOUT` (30m) and `HIVE_KICK_QUEUE_TURN_TIMEOUT` (60m). The optional `crux.KickQueuer` interface exposes it; `SendKick` and `SendKickAsync` are unchanged. Groundwork for meeting research kicks ([#18285](https://github.com/hivecommons/hive/issues/18285), parent [#18284](https://github.com/hivecommons/hive/issues/18284)). diff --git a/changelog.d/added-18286-kick-observers.md b/changelog.d/added-18286-kick-observers.md deleted file mode 100644 index 77a7041341..0000000000 --- a/changelog.d/added-18286-kick-observers.md +++ /dev/null @@ -1 +0,0 @@ -- Crux kick observers fan out: `Manager.AddKickObserver(func(crux.KickEvent)) (remove func())` registers any number of observers, each receiving every kick event in order on its own goroutine (a slow observer has events dropped rather than stalling the runtime). Events carry the kick ID of the queued kick they belong to, and the kick queue adds `kick-queued`, `kick-completed` and `kick-failed`. `SetKickObserver` keeps its single slot and its two events, so the Linear agent behaves exactly as before ([#18286](https://github.com/hivecommons/hive/issues/18286), parent [#18284](https://github.com/hivecommons/hive/issues/18284)). diff --git a/changelog.d/added-18287-output-signals.md b/changelog.d/added-18287-output-signals.md new file mode 100644 index 0000000000..0b5fbd5002 --- /dev/null +++ b/changelog.d/added-18287-output-signals.md @@ -0,0 +1 @@ +- Agent output markers now reach kick observers: a line that starts with `[STATUS]`, `[FINDING]`, `[COMPLETE]` or `[ERROR]` (after any CLI glyph) is delivered as a `output-signal` kick event carrying the agent, the in-flight queued kick's ID, the signal kind and the text after the marker (bounded to 500 runes), deduplicated per kick so a pane redraw does not repeat a finding. The existing log line is unchanged ([#18287](https://github.com/hivecommons/hive/issues/18287), parent [#18284](https://github.com/hivecommons/hive/issues/18284)). diff --git a/changelog.d/added-18288-research-pool.md b/changelog.d/added-18288-research-pool.md new file mode 100644 index 0000000000..6f52926cb4 --- /dev/null +++ b/changelog.d/added-18288-research-pool.md @@ -0,0 +1 @@ +- New `pkg/research` research agent pool for the meetings feature: `research-1..N` agents (`meetings.research_pool_size`, default 2, at most 8) defined as `ADVISORY` with no repo checkouts, so the proxy refuses every GitHub write whatever the prompt says. `Pool.Dispatch(ctx, query, meta)` queues the query on the least busy agent (starting it if stopped) and returns the kick ID; `Pool.Observe` reports `queued`, `running`, `finding`, `done` and `failed` per kick from the agent's `[FINDING]` / `[COMPLETE]` / `[ERROR]` output ([#18288](https://github.com/hivecommons/hive/issues/18288), parent [#18284](https://github.com/hivecommons/hive/issues/18284)). diff --git a/changelog.d/added-inbox-google-meet.md b/changelog.d/added-inbox-google-meet.md deleted file mode 100644 index 9b0d548a9a..0000000000 --- a/changelog.d/added-inbox-google-meet.md +++ /dev/null @@ -1 +0,0 @@ -- Inbox: Google Meet meetings as a knowledge integration — a `google-meet` connector (user-level OAuth with the hosted paste flow, or a service account with domain-wide delegation) that syncs conference records with a recording, transcript or Gemini notes into one fact each, with the event title, participants, Gemini notes as the summary and the transcript entries as `[mm:ss] Speaker: text` lines; a default `inbox.sources` entry with a Connect / Check / Sync now card (Meet recordings are Drive files behind a bearer token, so Inbox playback links the Drive page). diff --git a/changelog.d/added-inbox-transcription-auto-sweep.md b/changelog.d/added-inbox-transcription-auto-sweep.md new file mode 100644 index 0000000000..4ce6ca6e7f --- /dev/null +++ b/changelog.d/added-inbox-transcription-auto-sweep.md @@ -0,0 +1 @@ +- **Inbox: automatic transcription sweep for opted-in sources.** With `inbox.transcription.mode: auto`, the hive now transcribes new pending recordings by itself after each connector sync, but only from the sources listed in `inbox.transcription.sources`. The sweep queues only what the day's item cap and the month's minutes budget leave room for (items that do not fit stay pending, no skip records), never queues an item twice, retries failed items after 1h then 2h up to 3 runs, and retries budget skips the next UTC day. Runs are audited as `inbox_transcribe` by `inbox-auto-transcribe`. The ⚙ settings panel shows this month's minutes used, today's runs and the required egress (`api.deepgram.com:443`, only while transcription is enabled), and the network requirements docs list it. diff --git a/changelog.d/added-meetings-config-quorum-supervisor.md b/changelog.d/added-meetings-config-quorum-supervisor.md new file mode 100644 index 0000000000..55c674d1bf --- /dev/null +++ b/changelog.d/added-meetings-config-quorum-supervisor.md @@ -0,0 +1 @@ +- Meetings groundwork: a `meetings:` config block (off by default, every key overridable by a `HIVE_MEETINGS_*` env var, Vexa tenant key and Discord bot token kept in keystore-sealed secret files) and a supervisor that runs the Quorum meeting server as a loopback child process with a per-boot bearer token, health checks and restart backoff (#18289). diff --git a/changelog.d/changed-sessions-queue-deadline-clock-seam.md b/changelog.d/changed-sessions-queue-deadline-clock-seam.md new file mode 100644 index 0000000000..20ce7c6fa4 --- /dev/null +++ b/changelog.d/changed-sessions-queue-deadline-clock-seam.md @@ -0,0 +1 @@ +- Sessions: the queued-Send fail-open deadline is now measured through an injectable clock (no behaviour change; it makes the deadline test deterministic instead of racing a one-second wall-clock wait under CI load). diff --git a/changelog.d/fixed-18276-kick-busy-log-level.md b/changelog.d/fixed-18276-kick-busy-log-level.md new file mode 100644 index 0000000000..5ad3690840 --- /dev/null +++ b/changelog.d/fixed-18276-kick-busy-log-level.md @@ -0,0 +1 @@ +- A kick refused because another kick is already being typed into the same pane is now logged at info instead of warn, so this expected back-pressure no longer floods the fleet self-report as an unclassified repeated error ([#18276](https://github.com/hivecommons/hive/issues/18276)). diff --git a/src/Dockerfile b/src/Dockerfile index 243258c281..dc23fd65a8 100644 --- a/src/Dockerfile +++ b/src/Dockerfile @@ -447,6 +447,27 @@ RUN ARCH="${TARGETARCH:-$(dpkg --print-architecture)}" && \ echo "WARN: spektacular download failed - skipping (Inbox falls back to inbox.spek.agent)"; \ fi +# --- Layer 8.2: Quorum meeting MCP server (meetings, hivecommons/hive#18289) --- +# The pinned hivecommons/quorum release asset `quorum.mjs` (a single-file +# Node bundle), SHA-256-verified, installed to /usr/local/lib/quorum so the +# default meetings.quorum.binary resolves with no config. Quorum only runs +# when meetings.enabled is set. QUORUM_VERSION is empty until the first +# quorum release is cut; while it is empty the download is skipped and a +# hive with meetings enabled logs that the bundle is missing. A checksum +# mismatch hard-fails the build. Bump the version and digest together. +ARG QUORUM_VERSION="" +ARG QUORUM_SHA256="" +RUN if [ -n "${QUORUM_VERSION}" ]; then \ + test -n "${QUORUM_SHA256}" || { echo "QUORUM_SHA256 is required with QUORUM_VERSION" >&2; exit 1; } && \ + mkdir -p /usr/local/lib/quorum && \ + curl -fsSL --retry 5 --retry-delay 5 --retry-connrefused "https://github.com/hivecommons/quorum/releases/download/${QUORUM_VERSION}/quorum.mjs" -o /tmp/quorum.mjs && \ + echo "${QUORUM_SHA256} /tmp/quorum.mjs" | sha256sum -c - && \ + install -m 0644 /tmp/quorum.mjs /usr/local/lib/quorum/quorum.mjs && \ + rm -f /tmp/quorum.mjs; \ + else \ + echo "QUORUM_VERSION unset - skipping the quorum bundle (meetings unavailable in this image)"; \ + fi + # --- Layer 8.25: Google Antigravity CLI (backend: agy) --- # The vendor distributes a self-contained Linux executable, not an npm package. # Pin the version, build ID, and SHA-512 from its public updater manifest; do diff --git a/src/cmd/hive/boot.go b/src/cmd/hive/boot.go index 90c670773a..422f66eb28 100644 --- a/src/cmd/hive/boot.go +++ b/src/cmd/hive/boot.go @@ -345,6 +345,10 @@ type boot struct { // cruxSupervision is the sidecar runtime's child supervisor and probe // loop (hivecommons/hive#14138 stage 4); nil in-process. cruxSupervision *cruxRuntimeSupervision + + // meetingsQuorum is the Quorum child supervisor + // (hivecommons/hive#18289); nil when meetings are off. + meetingsQuorum quorumSupervisor } // deferStack stands in for the `defer` statements that used to sit in diff --git a/src/cmd/hive/boot_inbox.go b/src/cmd/hive/boot_inbox.go index e126471b11..f5273ce38c 100644 --- a/src/cmd/hive/boot_inbox.go +++ b/src/cmd/hive/boot_inbox.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "path/filepath" + "strings" "time" "github.com/hivecommons/hive/pkg/config" @@ -52,6 +53,11 @@ func (b *boot) bootInbox(baseDir string) { svc.EnableTranscription(inbox.NewTranscriptStore(filepath.Join(baseDir, "connector-state", "transcripts"))) b.inboxSvc = svc b.logger.Info("inbox started", "sources", len(srcs), "rules", len(b.cfg.Inbox.Rules)) + if tx := b.cfg.Inbox.Transcription; tx.Enabled { + // Transcription uploads audio from the hive process itself, so a + // locked-down spoke must allow this egress (network-requirements.md). + b.logger.Info("inbox transcription enabled", "mode", tx.EffectiveMode(), "provider", tx.EffectiveProvider(), "egress", strings.Join(inbox.TranscriptionEgressHosts(tx), ",")) + } } // inboxSources returns one source per configured voice type (inbox.sources) @@ -120,7 +126,22 @@ func (b *boot) inboxSource(baseDir, typ string, entries []config.KnowledgeConnec return facts } -// inboxAfterSync runs the auto-routing rules once a voice connector synced. +// inboxTranscribeSweep queues machine transcriptions for opted-in sources +// (inbox.transcription.mode: auto). Runs are audited as inbox_transcribe +// with the AutoTranscribeActor actor. +func (b *boot) inboxTranscribeSweep(ctx context.Context, svc *inbox.Service) { + opts := inbox.TranscribeOptions{Actor: inbox.AutoTranscribeActor, Audit: func(a inbox.TranscribeAudit) { + if b.dashSrv != nil { + b.dashSrv.AuditLog(a.Actor, "inbox_transcribe", a.Detail(), "") + } + }} + if n, err := svc.TranscribeSweep(ctx, opts); err != nil || n > 0 { + b.logger.Info("inbox transcription sweep", "queued", n, "error", err) + } +} + +// inboxAfterSync runs the auto-routing rules once a voice connector synced, +// then the transcription sweep. func (b *boot) inboxAfterSync(st connector.Status) { svc := b.inboxSvc if svc == nil { @@ -136,6 +157,7 @@ func (b *boot) inboxAfterSync(st connector.Status) { if n, err := svc.Sweep(ctx); err != nil { b.logger.Warn("inbox sweep", "error", err, "routed", n) } + b.inboxTranscribeSweep(ctx, svc) }() return } diff --git a/src/cmd/hive/main.go b/src/cmd/hive/main.go index 0b03740428..0c2235fde6 100644 --- a/src/cmd/hive/main.go +++ b/src/cmd/hive/main.go @@ -3803,6 +3803,12 @@ func (b *boot) bootDashboardAPIWith(deps bootDashboardAPIDeps) { b.dashSrv.SetSpektacularReconfigureFn(func() bool { return rewireSpektacular(b.cfg, b.dashSrv, b.logger, spekCloneAuth) }) + // Meetings (hivecommons/hive#18289): the Quorum child, only when + // meetings.enabled is set. Stopped on shutdown. + if sup := wireMeetings(b.ctx, b.cfg, b.logger); sup != nil { + b.meetingsQuorum = sup + b.cleanup.push(sup.Stop) + } // #8380: chain GitHub comments/labels behind the relay yank on takeover. ghClientFn := func() *github.Client { return b.ghClient } b.dashSrv.InstallClaimHooks(gfiClaimHooks(b.ctx, b.cfg, b.gfiState(), ghClientFn, b.dashSrv, b.logger, diff --git a/src/cmd/hive/meetingswire.go b/src/cmd/hive/meetingswire.go new file mode 100644 index 0000000000..699878e29f --- /dev/null +++ b/src/cmd/hive/meetingswire.go @@ -0,0 +1,86 @@ +package main + +import ( + "context" + "log/slog" + "os" + "path/filepath" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/meetings/quorumproc" +) + +// quorumSupervisor is the slice of *quorumproc.Supervisor boot uses. +type quorumSupervisor interface { + Start(ctx context.Context) error + Stop() +} + +// newQuorumSupervisor is the constructor seam; tests replace it. +var newQuorumSupervisor = func(opts quorumproc.Options) (quorumSupervisor, error) { + return quorumproc.New(opts) +} + +// meetingsQuorumOptions builds the Quorum child options from the meetings +// config, resolving the Vexa tenant key and the optional Discord bot token +// from the keystore. A secret that cannot be read is logged and left unset: +// Quorum then answers platform_unavailable for that platform instead of the +// hive failing to boot. +func meetingsQuorumOptions(ctx context.Context, m config.MeetingsConfig, logger *slog.Logger) quorumproc.Options { + vexaKey, err := m.VexaAPIKey(ctx) + if err != nil && logger != nil { + logger.Warn("[meetings] Vexa tenant key unreadable; meeting platforms other than Discord are unavailable", "error", err) + } + discordToken, err := m.DiscordToken(ctx) + if err != nil && logger != nil { + logger.Warn("[meetings] Discord bot token unreadable; Discord is unavailable", "error", err) + } + return quorumproc.Options{ + Binary: m.QuorumBinary(), + Listen: m.QuorumListen(), + VexaURL: m.VexaURL(), + VexaAPIKey: vexaKey, + DiscordToken: discordToken, + TranscribeURL: m.TranscribeURLOrEmpty(), + Logger: logger, + } +} + +// wireMeetings starts the Quorum child supervisor when meetings are enabled +// (hivecommons/hive#18289). With the feature off (the default) nothing is +// constructed and no process starts. It returns the running supervisor, or +// nil; the caller registers Stop for shutdown. +func wireMeetings(ctx context.Context, cfg *config.Config, logger *slog.Logger) quorumSupervisor { + if cfg == nil || !cfg.Meetings.IsEnabled() { + return nil + } + if bin := cfg.Meetings.QuorumBinary(); filepath.IsAbs(bin) { + if _, err := os.Stat(bin); err != nil { + if logger != nil { + logger.Error("[meetings] quorum bundle missing; meetings unavailable (this image was built without QUORUM_VERSION, or meetings.quorum.binary is wrong)", + "binary", bin, "error", err) + } + return nil + } + } + sup, err := newQuorumSupervisor(meetingsQuorumOptions(ctx, cfg.Meetings, logger)) + if err != nil { + if logger != nil { + logger.Error("[meetings] quorum supervisor not created; meetings unavailable", "error", err) + } + return nil + } + if err := sup.Start(ctx); err != nil { + if logger != nil { + logger.Error("[meetings] quorum supervisor did not start; meetings unavailable", "error", err) + } + sup.Stop() + return nil + } + if logger != nil { + logger.Info("[meetings] quorum supervisor started", + "binary", cfg.Meetings.QuorumBinary(), + "listen", cfg.Meetings.QuorumListen()) + } + return sup +} diff --git a/src/cmd/hive/meetingswire_test.go b/src/cmd/hive/meetingswire_test.go new file mode 100644 index 0000000000..df8a15dc25 --- /dev/null +++ b/src/cmd/hive/meetingswire_test.go @@ -0,0 +1,136 @@ +package main + +import ( + "bytes" + "context" + "errors" + "log/slog" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/meetings/quorumproc" +) + +type fakeQuorumSup struct { + startErr error + started bool + stopped bool +} + +func (f *fakeQuorumSup) Start(context.Context) error { + f.started = true + return f.startErr +} + +func (f *fakeQuorumSup) Stop() { f.stopped = true } + +func withQuorumSupervisor(t *testing.T, fn func(quorumproc.Options) (quorumSupervisor, error)) { + t.Helper() + prev := newQuorumSupervisor + newQuorumSupervisor = fn + t.Cleanup(func() { newQuorumSupervisor = prev }) +} + +func TestWireMeetingsOffByDefault(t *testing.T) { + withQuorumSupervisor(t, func(quorumproc.Options) (quorumSupervisor, error) { + t.Fatal("no supervisor may be built with meetings off") + return nil, nil + }) + if wireMeetings(context.Background(), nil, nil) != nil { + t.Fatal("nil config must not wire meetings") + } + if wireMeetings(context.Background(), &config.Config{}, nil) != nil { + t.Fatal("meetings must be off by default") + } +} + +func TestWireMeetingsStarts(t *testing.T) { + dir := t.TempDir() + keyFile := filepath.Join(dir, "vexa") + if err := os.WriteFile(keyFile, []byte("tenant\n"), 0o600); err != nil { + t.Fatal(err) + } + fake := &fakeQuorumSup{} + var got quorumproc.Options + withQuorumSupervisor(t, func(o quorumproc.Options) (quorumSupervisor, error) { + got = o + return fake, nil + }) + bin := filepath.Join(dir, "quorum.mjs") + if err := os.WriteFile(bin, []byte("//"), 0o600); err != nil { + t.Fatal(err) + } + cfg := &config.Config{Meetings: config.MeetingsConfig{ + Enabled: true, + Quorum: config.MeetingsQuorumConfig{Binary: bin}, + Vexa: config.MeetingsVexaConfig{URL: "https://vexa/", APIKeyFile: keyFile}, + Discord: config.MeetingsDiscordConfig{TokenFile: filepath.Join(dir, "missing")}, + }} + var logs bytes.Buffer + sup := wireMeetings(context.Background(), cfg, slog.New(slog.NewTextHandler(&logs, nil))) + if sup == nil || !fake.started { + t.Fatal("want a started supervisor") + } + if got.VexaAPIKey != "tenant" || got.VexaURL != "https://vexa" || got.DiscordToken != "" || + got.Listen != config.DefaultMeetingsQuorumListen || got.Binary != bin { + t.Fatalf("options %+v", got) + } + if strings.Contains(logs.String(), "tenant") { + t.Fatal("the tenant key must never be logged") + } +} + +func TestWireMeetingsFailures(t *testing.T) { + bin := filepath.Join(t.TempDir(), "quorum.mjs") + cfg := &config.Config{Meetings: config.MeetingsConfig{Enabled: true, Quorum: config.MeetingsQuorumConfig{Binary: bin}}} + withQuorumSupervisor(t, func(quorumproc.Options) (quorumSupervisor, error) { + t.Fatal("a missing bundle must not build a supervisor") + return nil, nil + }) + if wireMeetings(context.Background(), cfg, slog.New(slog.DiscardHandler)) != nil || + wireMeetings(context.Background(), cfg, nil) != nil { + t.Fatal("a missing bundle must leave meetings unwired") + } + if err := os.WriteFile(bin, []byte("//"), 0o600); err != nil { + t.Fatal(err) + } + withQuorumSupervisor(t, func(quorumproc.Options) (quorumSupervisor, error) { return nil, errors.New("boom") }) + if wireMeetings(context.Background(), cfg, slog.New(slog.DiscardHandler)) != nil { + t.Fatal("a construction error must leave meetings unwired") + } + fake := &fakeQuorumSup{startErr: errors.New("nope")} + withQuorumSupervisor(t, func(quorumproc.Options) (quorumSupervisor, error) { return fake, nil }) + if wireMeetings(context.Background(), cfg, slog.New(slog.DiscardHandler)) != nil || !fake.stopped { + t.Fatal("a start error must stop and drop the supervisor") + } + if wireMeetings(context.Background(), cfg, nil) != nil { + t.Fatal("nil logger path") + } +} + +func TestMeetingsQuorumOptionsUnreadableSecrets(t *testing.T) { + dir := t.TempDir() + m := config.MeetingsConfig{ + Vexa: config.MeetingsVexaConfig{APIKeyFile: dir}, + Discord: config.MeetingsDiscordConfig{TokenFile: dir}, + } + var logs bytes.Buffer + o := meetingsQuorumOptions(context.Background(), m, slog.New(slog.NewTextHandler(&logs, nil))) + if o.VexaAPIKey != "" || o.DiscordToken != "" { + t.Fatal("unreadable secrets must be left unset") + } + if !strings.Contains(logs.String(), "Vexa tenant key unreadable") || !strings.Contains(logs.String(), "Discord bot token unreadable") { + t.Fatalf("logs %q", logs.String()) + } +} + +func TestNewQuorumSupervisorDefault(t *testing.T) { + sup, err := newQuorumSupervisor(quorumproc.Options{Listen: "127.0.0.1:1", Binary: "/q", RuntimeDir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + sup.Stop() +} diff --git a/src/crux/agent/kick_chunk_pause_test.go b/src/crux/agent/kick_chunk_pause_test.go index 142091c93e..bcd1747a28 100644 --- a/src/crux/agent/kick_chunk_pause_test.go +++ b/src/crux/agent/kick_chunk_pause_test.go @@ -11,8 +11,14 @@ package agent // in-flight requests. import ( + "bytes" + "errors" + "log/slog" + "strings" "testing" "time" + + "github.com/hivecommons/hive/crux" ) // The lock is genuinely released for the duration of the pause: a second @@ -155,3 +161,23 @@ func TestKickDelivering_ClaimIsExclusive(t *testing.T) { t.Error("the pane was not reclaimable after delivery finished") } } + +// A refused concurrent delivery is expected back-pressure (callers retry on +// ErrAgentBusy), so it must not log at warn level and feed the selfreport. +func TestDeliverKickLocked_BusyPaneIsNotWarning(t *testing.T) { + var logBuf bytes.Buffer + m := NewManager(nil, slog.New(slog.NewTextHandler(&logBuf, nil)), ProjectContext{}) + agent := &AgentProcess{Name: "scanner"} + agent.kickDelivering.Store(true) + + m.mu.Lock() + err := m.deliverKickLocked(agent, "hello", "test") + m.mu.Unlock() + + if !errors.Is(err, crux.ErrAgentBusy) { + t.Fatalf("err = %v, want crux.ErrAgentBusy", err) + } + if strings.Contains(logBuf.String(), "level=WARN") { + t.Errorf("busy-pane skip logged at warn: %s", logBuf.String()) + } +} diff --git a/src/crux/agent/manager.go b/src/crux/agent/manager.go index 6c2d791294..cf86ec19d5 100644 --- a/src/crux/agent/manager.go +++ b/src/crux/agent/manager.go @@ -724,6 +724,9 @@ type Manager struct { // registered observer receives every kick event, with its kick ID, in // order on its own goroutine. See kick_observer.go. kickObservers kickObserverSet + // outputSignalSeen dedupes output-signal events per agent and kick + // (#18287); a pane redraw must not deliver a finding twice. + outputSignalSeen outputSignalDedupe // kickOutcomeObserver receives the verdict on how each kicked turn ended // (#7421); the governor consumes it. Same discipline as kickObserver. kickOutcomeObserver atomic.Pointer[func(agentName string, outcome KickOutcome)] @@ -739,13 +742,9 @@ type Manager struct { // manager lock and must not contend with the launch path. See kick_async.go. kickDispatches kickDispatchRegistry - // kickQueue holds the bounded per-agent FIFOs of ID-addressed kicks and - // their statuses (#18285). Own mutex, like kickDispatches. See - // kick_queue.go. + // kickQueue: per-agent FIFOs of ID-addressed kicks (#18285), own mutex. kickQueue kickQueue - // kickQueueDeliverFn / kickQueueIdleFn / kickQueuePollInterval are test - // overrides for the queue pump; nil / zero use the real delivery path, - // the real idle check and defaultKickQueuePollInterval. + // Queue pump test overrides; nil / zero use the real paths (kick_queue.go). kickQueueDeliverFn func(name, message string) error kickQueueIdleFn func(name string) bool kickQueuePollInterval time.Duration diff --git a/src/crux/agent/manager_kick.go b/src/crux/agent/manager_kick.go index 14f0f89473..a6ad125130 100644 --- a/src/crux/agent/manager_kick.go +++ b/src/crux/agent/manager_kick.go @@ -292,7 +292,9 @@ func (m *Manager) deliverKickLocked(agent *AgentProcess, message, trigger string // change the lock alone made it impossible. Two typists sharing one pane // would interleave their chunks into a single corrupt prompt. if !agent.kickDelivering.CompareAndSwap(false, true) { - m.logger.Warn("kick delivery skipped: another kick is already being typed into this pane", + // Expected back-pressure, not a defect: callers treat ErrAgentBusy + // as "retry later". Info keeps it out of the warn-level selfreport. + m.logger.Info("kick delivery skipped: another kick is already being typed into this pane", "agent", agent.Name, "trigger", trigger) return fmt.Errorf("%w: agent %s kick delivery already in progress", crux.ErrAgentBusy, agent.Name) } diff --git a/src/crux/agent/manager_poll.go b/src/crux/agent/manager_poll.go index f32bf2b7a2..8f9de3cfa2 100644 --- a/src/crux/agent/manager_poll.go +++ b/src/crux/agent/manager_poll.go @@ -523,8 +523,11 @@ var outputSignalPatterns = map[string]string{ } // logOutputSignals checks a line of agent output for meaningful patterns -// and emits a structured slog entry for each match. +// and emits a structured slog entry for each match. The structured markers +// ([STATUS], [FINDING], [COMPLETE], [ERROR]) also go to the kick observers +// (#18287; see output_signals.go). func (m *Manager) logOutputSignals(agent, line string) { + m.emitOutputSignal(agent, line) for pattern, event := range outputSignalPatterns { if strings.Contains(line, pattern) { preview := line diff --git a/src/crux/agent/output_signals.go b/src/crux/agent/output_signals.go new file mode 100644 index 0000000000..c024236f8e --- /dev/null +++ b/src/crux/agent/output_signals.go @@ -0,0 +1,113 @@ +package agent + +import ( + "strings" + "sync" + "unicode" + + "github.com/hivecommons/hive/crux" +) + +// Output signals to kick observers (#18287). logOutputSignals has always +// logged "[STATUS]", "[FINDING]", "[COMPLETE]" and "[ERROR]" lines; they now +// also reach the fan-out kick observers with the agent name, the in-flight +// queued kick's ID, the signal kind and the (bounded) text, so a caller that +// queued a kick can follow what it produces. + +const ( + // maxOutputSignalTextLen bounds Text and Detail on an output-signal + // event, in runes. + maxOutputSignalTextLen = 500 + // maxOutputSignalDedupe bounds the per-agent set of lines already + // delivered for the current kick; a pane redraw re-diffs old lines. + maxOutputSignalDedupe = 256 +) + +// outputSignalMarkers maps each delivered marker to its signal kind. +var outputSignalMarkers = []struct{ marker, signal string }{ + {"[STATUS]", crux.OutputSignalStatus}, + {"[FINDING]", crux.OutputSignalFinding}, + {"[COMPLETE]", crux.OutputSignalComplete}, + {"[ERROR]", crux.OutputSignalError}, +} + +// parseOutputSignal recognises a marker only as the first token of the line, +// after any leading CLI glyphs and whitespace ("⏺ [FINDING] x" counts). A +// marker mid-sentence (an echoed prompt explaining the convention, a quoted +// log) does not. +func parseOutputSignal(line string) (signal, text, whole string, ok bool) { + trimmed := strings.TrimLeftFunc(line, func(r rune) bool { + return r != '[' && !unicode.IsLetter(r) && !unicode.IsDigit(r) + }) + for _, mk := range outputSignalMarkers { + if rest, found := strings.CutPrefix(trimmed, mk.marker); found { + text = boundRunes(strings.TrimSpace(rest), maxOutputSignalTextLen) + whole = boundRunes(strings.TrimSpace(trimmed), maxOutputSignalTextLen) + return mk.signal, text, whole, true + } + } + return "", "", "", false +} + +// boundRunes cuts s to at most n runes. Unlike truncateStr it adds no +// ellipsis, so the bound is exact. +func boundRunes(s string, n int) string { + r := []rune(s) + if len(r) <= n { + return s + } + return string(r[:n]) +} + +// outputSignalDedupe remembers, per agent, which signal lines were already +// delivered for the agent's current kick ID. +type outputSignalDedupe struct { + mu sync.Mutex + byAgent map[string]*outputSignalSeen +} + +type outputSignalSeen struct { + kickID string + lines map[string]struct{} +} + +// firstSighting reports whether this line has not yet been delivered for +// the agent's kickID, and records it. +func (d *outputSignalDedupe) firstSighting(agent, kickID, line string) bool { + d.mu.Lock() + defer d.mu.Unlock() + if d.byAgent == nil { + d.byAgent = make(map[string]*outputSignalSeen) + } + seen := d.byAgent[agent] + if seen == nil || seen.kickID != kickID || len(seen.lines) >= maxOutputSignalDedupe { + seen = &outputSignalSeen{kickID: kickID, lines: make(map[string]struct{})} + d.byAgent[agent] = seen + } + if _, dup := seen.lines[line]; dup { + return false + } + seen.lines[line] = struct{}{} + return true +} + +// emitOutputSignal delivers a recognised marker line to the fan-out kick +// observers. Safe to call with m.mu held. +func (m *Manager) emitOutputSignal(agent, line string) { + signal, text, whole, ok := parseOutputSignal(line) + if !ok { + return + } + kickID := m.InFlightKickID(agent) + if !m.outputSignalSeen.firstSighting(agent, kickID, whole) { + return + } + m.emitKickEvent(crux.KickEvent{ + Agent: agent, + Event: crux.KickEventOutputSignal, + Detail: whole, + KickID: kickID, + Signal: signal, + Text: text, + }) +} diff --git a/src/crux/agent/output_signals_test.go b/src/crux/agent/output_signals_test.go new file mode 100644 index 0000000000..0dbfb22fc3 --- /dev/null +++ b/src/crux/agent/output_signals_test.go @@ -0,0 +1,93 @@ +package agent + +import ( + "strings" + "testing" + "time" + + "github.com/hivecommons/hive/crux" +) + +func TestParseOutputSignal(t *testing.T) { + cases := []struct { + line, signal, text string + ok bool + }{ + {"[FINDING] vLLM 0.9 adds prefix caching", crux.OutputSignalFinding, "vLLM 0.9 adds prefix caching", true}, + {"⏺ [COMPLETE] three sources agree", crux.OutputSignalComplete, "three sources agree", true}, + {" [STATUS] reading the docs", crux.OutputSignalStatus, "reading the docs", true}, + {"● [ERROR] site unreachable", crux.OutputSignalError, "site unreachable", true}, + {"[FINDING]", crux.OutputSignalFinding, "", true}, + {"emit lines that start with [FINDING] for each result", "", "", false}, + {"[HEARTBEAT] alive", "", "", false}, + {"nothing here", "", "", false}, + } + for _, c := range cases { + signal, text, _, ok := parseOutputSignal(c.line) + if ok != c.ok || signal != c.signal || text != c.text { + t.Errorf("parseOutputSignal(%q) = (%q, %q, %v), want (%q, %q, %v)", c.line, signal, text, ok, c.signal, c.text, c.ok) + } + } + _, text, whole, _ := parseOutputSignal("[FINDING] " + strings.Repeat("x", 2*maxOutputSignalTextLen)) + if n := len([]rune(text)); n > maxOutputSignalTextLen { + t.Errorf("text is %d runes, want at most %d", n, maxOutputSignalTextLen) + } + if n := len([]rune(whole)); n > maxOutputSignalTextLen { + t.Errorf("whole line is %d runes, want at most %d", n, maxOutputSignalTextLen) + } +} + +func TestLogOutputSignals_DeliversToObserversWithKickID(t *testing.T) { + m := newKickQueueManager(t, func(string) bool { return true }, func(string, string) error { return nil }) + events := make(chan crux.KickEvent, 16) + defer m.AddKickObserver(func(ev crux.KickEvent) { + if ev.Event == crux.KickEventOutputSignal { + events <- ev + } + })() + + // Outside any queued kick: delivered with an empty kick id. + m.logOutputSignals(kqTestAgent, "[STATUS] warming up") + ev := recvKickEvent(t, events) + if ev.Signal != crux.OutputSignalStatus || ev.KickID != "" || ev.Agent != kqTestAgent || ev.Text != "warming up" || ev.Detail != "[STATUS] warming up" { + t.Fatalf("status event = %+v", ev) + } + + id, _ := m.EnqueueKick(kqTestAgent, "meet-1.r1", "research") + waitKickPhase(t, m, id, crux.KickPhaseDelivered) + m.logOutputSignals(kqTestAgent, "⏺ [FINDING] answer one") + m.logOutputSignals(kqTestAgent, "⏺ [FINDING] answer one") // pane redraw + m.logOutputSignals(kqTestAgent, "git push origin main") // logged only + m.logOutputSignals(kqTestAgent, "[COMPLETE] done") + first, second := recvKickEvent(t, events), recvKickEvent(t, events) + if first.Signal != crux.OutputSignalFinding || first.KickID != id || first.Text != "answer one" { + t.Fatalf("finding event = %+v", first) + } + if second.Signal != crux.OutputSignalComplete || second.KickID != id { + t.Fatalf("second event = %+v, want the complete signal (the redraw must be deduplicated)", second) + } + select { + case extra := <-events: + t.Fatalf("unexpected extra event %+v", extra) + case <-time.After(50 * time.Millisecond): + } +} + +func TestOutputSignalDedupe_ResetsPerKickAndBound(t *testing.T) { + var d outputSignalDedupe + if !d.firstSighting("a", "k1", "[FINDING] x") || d.firstSighting("a", "k1", "[FINDING] x") { + t.Fatal("same line within one kick must be delivered once") + } + if !d.firstSighting("a", "k2", "[FINDING] x") { + t.Fatal("a new kick must reset the dedupe set") + } + if !d.firstSighting("b", "k2", "[FINDING] x") { + t.Fatal("dedupe must be per agent") + } + for i := 0; i < maxOutputSignalDedupe; i++ { + d.firstSighting("c", "k", strings.Repeat("y", i+1)) + } + if !d.firstSighting("c", "k", "y") { + t.Fatal("a full dedupe set must reset rather than grow") + } +} diff --git a/src/crux/kick_event.go b/src/crux/kick_event.go index a71e9f1b70..62cbaa2aa3 100644 --- a/src/crux/kick_event.go +++ b/src/crux/kick_event.go @@ -21,6 +21,23 @@ const ( // KickEventFailed: a queued kick will not be delivered or its turn was // abandoned. Detail is the reason. KickEventFailed = "kick-failed" + // KickEventOutputSignal: the agent printed a structured output marker + // (#18287). Signal names which, Text is the line after the marker and + // Detail the whole line, both length-bounded. + KickEventOutputSignal = "output-signal" +) + +// Output signals: the structured markers an agent prints to report progress +// (#18287). Each is recognised only as the first token of an output line. +const ( + // OutputSignalStatus: "[STATUS] " progress note. + OutputSignalStatus = "status" + // OutputSignalFinding: "[FINDING] " result fragment. + OutputSignalFinding = "finding" + // OutputSignalComplete: "[COMPLETE] " the task is done. + OutputSignalComplete = "complete" + // OutputSignalError: "[ERROR] " the task failed. + OutputSignalError = "error" ) // KickEvent is one kick lifecycle event as fan-out observers receive it. @@ -34,6 +51,11 @@ type KickEvent struct { // KickID is the queued kick the event belongs to, or "" for a kick that // did not come through the kick queue. KickID string `json:"kickId,omitempty"` + // Signal is the OutputSignal* kind for a KickEventOutputSignal event. + Signal string `json:"signal,omitempty"` + // Text is the payload after the marker for a KickEventOutputSignal + // event. + Text string `json:"text,omitempty"` // At is when the runtime emitted the event. At time.Time `json:"at"` } diff --git a/src/docs/cli-pins.md b/src/docs/cli-pins.md index c49b7a0fd1..c6cf0044e8 100644 --- a/src/docs/cli-pins.md +++ b/src/docs/cli-pins.md @@ -11,6 +11,7 @@ Every agent CLI hive drives is baked into the images at a fixed version: | Goose | `GOOSE_VERSION`, `GOOSE_SHA256_{AMD64,ARM64}` | GitHub releases `aaif-goose/goose` | SHA-256 per arch | | Antigravity (agy) | `AGY_VERSION` + `AGY_BUILD`, `AGY_SHA512_*` (hub); `AGY_VERSION=-`, `AGY_SHA256_*` (contributor) | Google's updater artifact, as tracked by the Homebrew `antigravity-cli` cask | SHA-512 (hub) / SHA-256 (contributor) per arch | | spektacular | `SPEKTACULAR_VERSION`, `SPEKTACULAR_SHA256_{AMD64,ARM64}` | GitHub releases `hivecommons/spektacular` | SHA-256 per arch, from the release `checksums.txt` (manual bump for now; not yet in `cli-pin-bump.sh`) | +| quorum (meetings bundle, not a CLI backend) | `QUORUM_VERSION`, `QUORUM_SHA256` | GitHub releases `hivecommons/quorum` (`quorum.mjs` asset) | SHA-256 of the single arch-independent bundle; empty version skips the download (manual bump) | | Oh My Pi (omp) | `OMP_VERSION`, `OMP_SHA256_*` | GitHub releases `can1357/oh-my-pi` | SHA-256 per arch, cross-checked with `SHA256SUMS.txt` | | Muse Code | `MUSE_VERSION`, `MUSE_SHA256_*` | `api.meta.ai/muse-code/channels/muse-stable` and its release manifest | SHA-256 per arch, cross-checked with the manifest | | bobshell (bob) | `BOBSHELL_VERSION`, `BOBSHELL_SHA256_*` | `bobshell2-version.txt` on IBM COS | SHA-256, cross-checked with the vendor `.tgz.sha256` | diff --git a/src/docs/env-vars.md b/src/docs/env-vars.md index bfa22f7860..340fa549d6 100644 --- a/src/docs/env-vars.md +++ b/src/docs/env-vars.md @@ -362,6 +362,37 @@ Part 2 of [RFC #4492](https://github.com/hivecommons/hive/issues/4492): the hive Inside an **agent** session (set by the hive, never by the operator): ISSUES_ONLY+ agents receive `LINEAR_ACCESS_TOKEN` (the connected app's OAuth token, `Authorization: Bearer`) or, when no workspace is connected, `LINEAR_API_KEY` (the work-source key, bare `Authorization`). Advisory agents receive neither and have both stripped. See [linear-agent.md](linear-agent.md#github-issue-parity-agents-writing-to-linear). +## Meetings (Quorum) + +Hive as a live meeting participant ([#18284](https://github.com/hivecommons/hive/issues/18284)). Each variable overrides the matching `meetings.*` key in `hive.yaml`; with `meetings.enabled` off nothing starts. The default consent notice is: "Hive has joined to take notes and help with research. It records this meeting's transcript. Type 'hive leave' to remove it." Other `QUORUM_*` variables set on hive (the contract's numeric limits, for example `QUORUM_CHAT_MAX_CHARS`) pass through to the Quorum child unchanged. + +| Variable | Required | Default | Purpose | +|---|---:|---|---| +| `HIVE_MEETINGS_ENABLED` | No | `meetings.enabled` (off) | Turns meetings on: hive starts the Quorum child and can join live meetings. Accepts `1/true/yes/on` and `0/false/no/off`. | +| `HIVE_MEETINGS_QUORUM_BINARY` | No | `/usr/local/lib/quorum/quorum.mjs` | Quorum bundle (run through `node` when it ends in `.mjs` or `.js`) or executable. Overrides `meetings.quorum.binary`. | +| `HIVE_MEETINGS_QUORUM_LISTEN` | No | `127.0.0.1:8765` | Loopback address Quorum listens on (passed as `QUORUM_LISTEN`). Non-loopback values are rejected. | +| `HIVE_MEETINGS_VEXA_URL` | Yes for Meet, Teams and Zoom | none | Vexa gateway base URL (passed as `QUORUM_VEXA_URL`). Overrides `meetings.vexa.url`. | +| `HIVE_MEETINGS_VEXA_API_KEY` | No | none | Vexa tenant key. Secret. Wins over the keystore-sealed secret file; prefer the file, which the dashboard writes. | +| `HIVE_MEETINGS_VEXA_API_KEY_FILE` | No | `/data/secrets/meetings_vexa_api_key` | Keystore-sealed secret file holding the Vexa tenant key. Overrides `meetings.vexa.api_key_file`. | +| `HIVE_MEETINGS_DISCORD_TOKEN` | No | none | Per-hive Discord bot token. Secret. Wins over the secret file. Without a token Discord is unavailable. | +| `HIVE_MEETINGS_DISCORD_TOKEN_FILE` | No | `/data/secrets/meetings_discord_token` | Keystore-sealed secret file holding the Discord bot token. Overrides `meetings.discord.token_file`. | +| `HIVE_MEETINGS_TRANSCRIBE_URL` | No | none | Shared transcription service for Discord audio (passed as `QUORUM_TRANSCRIBE_URL`). | +| `HIVE_MEETINGS_DELIVERY` | No | `chat` | Default delivery for meeting output. Only `chat` is supported. | +| `HIVE_MEETINGS_VOICE_ON_ADDRESS` | No | `true` | Lets hive speak when addressed by name in a meeting whose voice is enabled; otherwise it answers in chat. | +| `HIVE_MEETINGS_BOT_NAME` | No | `hive (notes)` | Display name the meeting bot joins with. | +| `HIVE_MEETINGS_INVITE_POLICY` | No | `owner` | Who may send hive to a meeting. Only `owner` is supported. | +| `HIVE_MEETINGS_RESEARCH_POOL_SIZE` | No | `2` | Number of research agents (`research-1..N`). | +| `HIVE_MEETINGS_RESEARCH_PER_MEETING` | No | `5` | Research items one meeting may start before each further item needs dashboard approval. | +| `HIVE_MEETINGS_CHAT_POSTS_PER_MINUTE` | No | `6` | Cap on hive's chat posts into one meeting. | +| `HIVE_MEETINGS_LOBBY_TIMEOUT_S` | No | `600` | Seconds hive waits in a meeting lobby before leaving. | +| `HIVE_MEETINGS_RETENTION_DAYS` | No | `30` | Days an ended meeting's transcript is kept before it is purged. | +| `HIVE_MEETINGS_CONSENT_TEXT` | No | the notice above | Notice posted in the meeting chat on join, before anything else. At most 2000 characters. | +| `HIVE_MEETINGS_DATA_DIR` | No | `/data/meetings` | Where meeting records and transcripts are stored. | +| `HIVE_MEETINGS_MAX_TOKENS_PER_MEETING` | No | `200000` | Per-meeting LLM token budget for Spektacular live mode. | +| `HIVE_MEETINGS_MAX_TOKENS_PER_HOUR` | No | `500000` | Hive-wide hourly token ceiling across meetings. | +| `HIVE_MEETINGS_MAX_TOKENS_PER_DAY` | No | `2000000` | Hive-wide daily token ceiling across meetings. | +| `HIVE_MEETINGS_MAX_INTENTS_PER_MEETING` | No | `500` | Per-meeting budget counted in Spek intents when Spek reports no token usage. | + ## Backend launch controls | Variable | Required | Default | Purpose | diff --git a/src/docs/inbox-transcription.md b/src/docs/inbox-transcription.md index 80c496876f..f8baa607e3 100644 --- a/src/docs/inbox-transcription.md +++ b/src/docs/inbox-transcription.md @@ -112,6 +112,38 @@ and the failure reason). Once done, the item is no longer pending and its deepgram/nova-3)**, in the same `[mm:ss] Speaker N: text` line shape as Plaud and meeting transcripts. +## Automatic transcription (auto mode) + +With `mode: auto`, the hive also transcribes new pending items by itself, +but only from the sources listed in `inbox.transcription.sources` (an Inbox +source type such as `plaud`, or a `knowledge.connectors[]` entry name). An +empty list transcribes nothing automatically. The sweep runs after each +connector sync, right after the auto-routing rules: + +- It queues only what today's `daily_max_items` and this month's + `monthly_budget_minutes` leave room for; items that do not fit stay + pending, without a skip record, and are picked up once budget frees up. +- Runs still execute one at a time. An item that is queued, running or done + is never queued again, and a recording listed twice is queued once. +- A **failed** item is retried after 1 hour, then 2 hours, up to 3 runs in + total. An item **skipped for budget** is retried once the UTC day turns. + Items skipped as too long, too large or unsupported are not retried + automatically. The owner can always click **Retry transcription**. +- Sweep-started runs are audited as `inbox_transcribe` with the actor + `inbox-auto-transcribe`. + +The ⚙ settings panel shows this month's minutes used against the budget, +today's runs against the daily limit, and the egress the hive needs. + +## Egress + +The hive process itself uploads the audio, so a spoke behind an egress +allowlist must allow `api.deepgram.com:443` while transcription is enabled +(and the source's audio host, which playback already needs). Nothing is +needed while transcription is off. `settings.transcription.egress_hosts` and +the boot log line `inbox transcription enabled` name the host. See +[network-requirements.md](network-requirements.md). + ## Upstream text wins The machine transcript is a fallback. If the source later produces its own diff --git a/src/docs/manual-provisioning.md b/src/docs/manual-provisioning.md index 1b7d493f12..e46b8cafd8 100644 --- a/src/docs/manual-provisioning.md +++ b/src/docs/manual-provisioning.md @@ -818,6 +818,9 @@ A standalone hive needs **no hub egress**. The minimal outbound allowlist is: - **Your inference endpoint** — if it's in-cluster (the shipped `src/deploy/inference` backend, or your own litellm/vllm Service) this is **cluster-internal traffic, not external egress** at all. +- **`api.deepgram.com:443`** — only if you enable Inbox transcription + (`inbox.transcription.enabled`); see + [inbox-transcription.md](inbox-transcription.md). Hub hosts (`hub.kubestellar.io`) are **not** required. For the full egress matrix (ports, hosts, when each is needed) see diff --git a/src/docs/network-requirements.md b/src/docs/network-requirements.md index ddb1af9e3e..9135566408 100644 --- a/src/docs/network-requirements.md +++ b/src/docs/network-requirements.md @@ -43,6 +43,7 @@ Reverse proxies, Routes, and Ingresses must preserve HTTP upgrade headers and lo | Kubernetes API | In-cluster spokes/hub | Used for health/provisioning/route discovery when those features are configured. | | Login provider issuers | Hub with multi-provider login | GitHub OAuth endpoints, plus the OIDC issuer/JWKS hosts for each configured provider (e.g. `accounts.google.com`, `login.microsoftonline.com`, `sso.redhat.com`, or your IBMid/custom issuer). | | OCI Object Storage (`objectstorage..oraclecloud.com`) | Hub running DR backups | Only when the opt-in backup CronJob is deployed; see [backup-restore.md](backup-restore.md). | +| `api.deepgram.com:443` | Spokes with Inbox transcription | Only when `inbox.transcription.enabled` is true. The hive process itself (not an agent) uploads recording audio to Deepgram under the owner's own key, so it does not go through the agent MITM proxy; the hive also downloads the audio from the source's audio host (for Plaud, its presigned CDN link). See [inbox-transcription.md](inbox-transcription.md). | ## Egress through a corporate forward proxy diff --git a/src/pkg/config/config.go b/src/pkg/config/config.go index eeda2933ff..8937b686b7 100644 --- a/src/pkg/config/config.go +++ b/src/pkg/config/config.go @@ -160,8 +160,11 @@ type Config struct { Jev JevConfig `yaml:"jev,omitempty" json:"jev,omitempty"` // Runs tunes long-running runs and the opt-in Spektacular stage runner // (hivecommons/hive#8303). Zero value: runner off, default retry budget. - Runs RunsConfig `yaml:"runs,omitempty" json:"runs,omitempty"` - Retro RetroConfig `yaml:"retro,omitempty" json:"retro,omitempty"` + Runs RunsConfig `yaml:"runs,omitempty" json:"runs,omitempty"` + Retro RetroConfig `yaml:"retro,omitempty" json:"retro,omitempty"` + // Meetings turns on hive as a live meeting participant through Quorum + // (hivecommons/hive#18284). Zero value: off, no Quorum process. + Meetings MeetingsConfig `yaml:"meetings,omitempty" json:"meetings,omitempty"` Swarm SwarmConfig `yaml:"swarm,omitempty" json:"swarm,omitempty"` Autonomy AutonomyConfig `yaml:"autonomy,omitempty" json:"autonomy,omitempty"` Review ReviewConfig `yaml:"review,omitempty" json:"review,omitempty"` diff --git a/src/pkg/config/meetings_config.go b/src/pkg/config/meetings_config.go new file mode 100644 index 0000000000..5c7fc7680c --- /dev/null +++ b/src/pkg/config/meetings_config.go @@ -0,0 +1,487 @@ +package config + +import ( + "context" + "errors" + "fmt" + "net" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/hivecommons/hive/crux/keystore" +) + +// Meetings: hive as a live meeting participant through Quorum +// (hivecommons/hive#18284, hivecommons/quorum docs/contract.md, +// quorum.contract/v1). The whole feature is OFF until meetings.enabled is +// set; with it off nothing is constructed and no Quorum process starts. +// +// Every value below can be overridden by an environment variable, read at +// call time so a hosted hive's operator can turn meetings on from the +// deployment without editing hive.yaml. Precedence is env, then hive.yaml, +// then the default. Secrets (the Vexa tenant key and the optional Discord bot +// token) are never stored in hive.yaml: they live in keystore-sealed secret +// files under /data/secrets, or arrive through an environment variable. + +const ( + // DefaultMeetingsQuorumBinary is where the hive image installs the pinned + // Quorum bundle. A path ending in .mjs or .js runs through node. + DefaultMeetingsQuorumBinary = "/usr/local/lib/quorum/quorum.mjs" + // DefaultMeetingsQuorumListen is the loopback address the Quorum child + // listens on (the contract's QUORUM_LISTEN default). + DefaultMeetingsQuorumListen = "127.0.0.1:8765" + // DefaultMeetingsDelivery is where meeting output is delivered by + // default: the meeting chat, so hive never talks over people. + DefaultMeetingsDelivery = MeetingsDeliveryChat + // DefaultMeetingsBotName is the display name the meeting bot joins with. + // It says the bot takes notes. + DefaultMeetingsBotName = "hive (notes)" + // DefaultMeetingsInvitePolicy lets only the hive owner send hive to a + // meeting. + DefaultMeetingsInvitePolicy = MeetingsInvitePolicyOwner + // DefaultMeetingsResearchPoolSize is the number of research agents + // (research-1..N) created when meetings are enabled. + DefaultMeetingsResearchPoolSize = 2 + // DefaultMeetingsResearchPerMeeting caps research items one meeting may + // start without dashboard approval. + DefaultMeetingsResearchPerMeeting = 5 + // DefaultMeetingsChatPostsPerMinute caps hive's chat posts into one + // meeting. + DefaultMeetingsChatPostsPerMinute = 6 + // DefaultMeetingsLobbyTimeoutS is how long hive waits in a meeting + // lobby before it gives up and leaves. + DefaultMeetingsLobbyTimeoutS = 600 + // DefaultMeetingsRetentionDays is how long hive keeps an ended meeting's + // transcript before the retention sweeper purges it. + DefaultMeetingsRetentionDays = 30 + // DefaultMeetingsMaxTokensPerMeeting is the per-meeting LLM token budget + // for Spektacular live mode, when Spek reports usage. + DefaultMeetingsMaxTokensPerMeeting = 200000 + // DefaultMeetingsMaxTokensPerHour is the hive-wide hourly ceiling across + // all meetings. + DefaultMeetingsMaxTokensPerHour = 500000 + // DefaultMeetingsMaxTokensPerDay is the hive-wide daily ceiling across + // all meetings. + DefaultMeetingsMaxTokensPerDay = 2000000 + // DefaultMeetingsMaxIntentsPerMeeting is the per-meeting budget counted + // in intents when Spek does not report token usage. + DefaultMeetingsMaxIntentsPerMeeting = 500 + // DefaultMeetingsConsentText is posted in the meeting chat on join, + // before anything else. + DefaultMeetingsConsentText = "Hive has joined to take notes and help with research. It records this meeting's transcript. Type 'hive leave' to remove it." + // DefaultMeetingsDataDir is where meeting records and transcripts live. + DefaultMeetingsDataDir = "/data/meetings" + // DefaultMeetingsVexaAPIKeyFile is the keystore-sealed secret file that + // holds the Vexa tenant key. + DefaultMeetingsVexaAPIKeyFile = WritableSecretsDir + "/meetings_vexa_api_key" + // DefaultMeetingsDiscordTokenFile is the keystore-sealed secret file that + // holds the optional per-hive Discord bot token. + DefaultMeetingsDiscordTokenFile = WritableSecretsDir + "/meetings_discord_token" + + // MeetingsDeliveryChat posts meeting output in the meeting chat. + MeetingsDeliveryChat = "chat" + // MeetingsInvitePolicyOwner lets only the hive owner invite hive. + MeetingsInvitePolicyOwner = "owner" + + // meetingsMaxConsentChars mirrors the contract's QUORUM_CHAT_MAX_CHARS + // default: the consent notice is one chat post. + meetingsMaxConsentChars = 2000 + // meetingsMaxBotNameChars bounds the display name. + meetingsMaxBotNameChars = 64 +) + +// Environment overrides for the meetings block. Each one wins over hive.yaml. +const ( + MeetingsEnabledEnv = "HIVE_MEETINGS_ENABLED" + MeetingsQuorumBinaryEnv = "HIVE_MEETINGS_QUORUM_BINARY" + MeetingsQuorumListenEnv = "HIVE_MEETINGS_QUORUM_LISTEN" + MeetingsVexaURLEnv = "HIVE_MEETINGS_VEXA_URL" + MeetingsVexaAPIKeyEnv = "HIVE_MEETINGS_VEXA_API_KEY" + MeetingsVexaAPIKeyFileEnv = "HIVE_MEETINGS_VEXA_API_KEY_FILE" + MeetingsDiscordTokenEnv = "HIVE_MEETINGS_DISCORD_TOKEN" + MeetingsDiscordTokenFileEnv = "HIVE_MEETINGS_DISCORD_TOKEN_FILE" + MeetingsTranscribeURLEnv = "HIVE_MEETINGS_TRANSCRIBE_URL" + MeetingsDeliveryEnv = "HIVE_MEETINGS_DELIVERY" + MeetingsVoiceOnAddressEnv = "HIVE_MEETINGS_VOICE_ON_ADDRESS" + MeetingsBotNameEnv = "HIVE_MEETINGS_BOT_NAME" + MeetingsInvitePolicyEnv = "HIVE_MEETINGS_INVITE_POLICY" + MeetingsResearchPoolSizeEnv = "HIVE_MEETINGS_RESEARCH_POOL_SIZE" + MeetingsResearchPerMeetingEnv = "HIVE_MEETINGS_RESEARCH_PER_MEETING" + MeetingsChatPostsPerMinuteEnv = "HIVE_MEETINGS_CHAT_POSTS_PER_MINUTE" + MeetingsLobbyTimeoutEnv = "HIVE_MEETINGS_LOBBY_TIMEOUT_S" + MeetingsRetentionDaysEnv = "HIVE_MEETINGS_RETENTION_DAYS" + MeetingsConsentTextEnv = "HIVE_MEETINGS_CONSENT_TEXT" + MeetingsDataDirEnv = "HIVE_MEETINGS_DATA_DIR" + MeetingsTokensPerMeetingEnv = "HIVE_MEETINGS_MAX_TOKENS_PER_MEETING" + MeetingsTokensPerHourEnv = "HIVE_MEETINGS_MAX_TOKENS_PER_HOUR" + MeetingsTokensPerDayEnv = "HIVE_MEETINGS_MAX_TOKENS_PER_DAY" + MeetingsIntentsPerMeetingEnv = "HIVE_MEETINGS_MAX_INTENTS_PER_MEETING" +) + +// MeetingsConfig is the `meetings:` block. The zero value is the feature off. +type MeetingsConfig struct { + // Enabled turns meetings on. Default false. + Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + // Quorum configures the Quorum child process. + Quorum MeetingsQuorumConfig `yaml:"quorum,omitempty" json:"quorum,omitempty"` + // Vexa configures the Vexa meeting-bot gateway Quorum talks to. + Vexa MeetingsVexaConfig `yaml:"vexa,omitempty" json:"vexa,omitempty"` + // Discord configures the optional Discord voice adapter. + Discord MeetingsDiscordConfig `yaml:"discord,omitempty" json:"discord,omitempty"` + // TranscribeURL is the shared transcription service Quorum sends + // Discord audio to (QUORUM_TRANSCRIBE_URL). Empty leaves it unset. + TranscribeURL string `yaml:"transcribe_url,omitempty" json:"transcribe_url,omitempty"` + // Delivery is where meeting output goes by default. Only "chat". + Delivery string `yaml:"delivery,omitempty" json:"delivery,omitempty"` + // VoiceOnAddress lets hive speak when the meeting addresses it by name + // and the meeting has voice enabled. Nil defaults true. + VoiceOnAddress *bool `yaml:"voice_on_address,omitempty" json:"voice_on_address,omitempty"` + // BotName is the display name the bot joins with. + BotName string `yaml:"bot_name,omitempty" json:"bot_name,omitempty"` + // InvitePolicy is who may send hive to a meeting. Only "owner". + InvitePolicy string `yaml:"invite_policy,omitempty" json:"invite_policy,omitempty"` + // ResearchPoolSize is the number of research agents. Zero or negative + // means DefaultMeetingsResearchPoolSize. + ResearchPoolSize int `yaml:"research_pool_size,omitempty" json:"research_pool_size,omitempty"` + // ResearchPerMeeting caps research items per meeting before dashboard + // approval is required. Zero or negative means the default. + ResearchPerMeeting int `yaml:"research_per_meeting,omitempty" json:"research_per_meeting,omitempty"` + // ChatPostsPerMinute caps hive's chat posts into one meeting. Zero or + // negative means the default. + ChatPostsPerMinute int `yaml:"chat_posts_per_minute,omitempty" json:"chat_posts_per_minute,omitempty"` + // LobbyTimeoutS is how long hive waits in a lobby. Zero or negative + // means the default. + LobbyTimeoutS int `yaml:"lobby_timeout_s,omitempty" json:"lobby_timeout_s,omitempty"` + // Budget bounds Spektacular live-mode LLM use. + Budget MeetingsBudgetConfig `yaml:"budget,omitempty" json:"budget,omitempty"` + // RetentionDays is how long an ended meeting's transcript is kept. Zero + // or negative means the default. + RetentionDays int `yaml:"retention_days,omitempty" json:"retention_days,omitempty"` + // ConsentText is the notice posted in the meeting chat on join. + ConsentText string `yaml:"consent_text,omitempty" json:"consent_text,omitempty"` + // DataDir is where meeting records and transcripts are stored. + DataDir string `yaml:"data_dir,omitempty" json:"data_dir,omitempty"` +} + +// MeetingsQuorumConfig is meetings.quorum. +type MeetingsQuorumConfig struct { + // Binary is the Quorum bundle (run through node when it ends in .mjs or + // .js) or an executable. + Binary string `yaml:"binary,omitempty" json:"binary,omitempty"` + // Listen is the loopback host:port Quorum listens on. + Listen string `yaml:"listen,omitempty" json:"listen,omitempty"` +} + +// MeetingsVexaConfig is meetings.vexa. +type MeetingsVexaConfig struct { + // URL is the Vexa gateway base URL (QUORUM_VEXA_URL). + URL string `yaml:"url,omitempty" json:"url,omitempty"` + // APIKeyFile is the keystore-sealed secret file holding the tenant key. + // A path, never the key. + APIKeyFile string `yaml:"api_key_file,omitempty" json:"api_key_file,omitempty"` +} + +// MeetingsDiscordConfig is meetings.discord. +type MeetingsDiscordConfig struct { + // TokenFile is the keystore-sealed secret file holding the bot token. + // A path, never the token. + TokenFile string `yaml:"token_file,omitempty" json:"token_file,omitempty"` +} + +// MeetingsBudgetConfig is meetings.budget. Zero or negative means the +// default for each field. +type MeetingsBudgetConfig struct { + MaxTokensPerMeeting int `yaml:"max_tokens_per_meeting,omitempty" json:"max_tokens_per_meeting,omitempty"` + MaxTokensPerHour int `yaml:"max_tokens_per_hour,omitempty" json:"max_tokens_per_hour,omitempty"` + MaxTokensPerDay int `yaml:"max_tokens_per_day,omitempty" json:"max_tokens_per_day,omitempty"` + MaxIntentsPerMeeting int `yaml:"max_intents_per_meeting,omitempty" json:"max_intents_per_meeting,omitempty"` +} + +// meetingsEnvString returns the trimmed env value when set, else the +// trimmed config value, else def. +func meetingsEnvString(env, value, def string) string { + if v := strings.TrimSpace(os.Getenv(env)); v != "" { + return v + } + if v := strings.TrimSpace(value); v != "" { + return v + } + return def +} + +// meetingsEnvInt returns a positive env value when set and valid, else a +// positive config value, else def. +func meetingsEnvInt(env string, value, def int) int { + if raw := strings.TrimSpace(os.Getenv(env)); raw != "" { + if n, err := strconv.Atoi(raw); err == nil && n > 0 { + return n + } + } + if value > 0 { + return value + } + return def +} + +// IsEnabled reports whether meetings are on: HIVE_MEETINGS_ENABLED, then +// meetings.enabled. +func (m MeetingsConfig) IsEnabled() bool { + if v, ok := parseBoolEnv(MeetingsEnabledEnv); ok { + return v + } + return m.Enabled +} + +// QuorumBinary returns the Quorum bundle or executable path. +func (m MeetingsConfig) QuorumBinary() string { + return meetingsEnvString(MeetingsQuorumBinaryEnv, m.Quorum.Binary, DefaultMeetingsQuorumBinary) +} + +// QuorumListen returns the Quorum listen address. +func (m MeetingsConfig) QuorumListen() string { + return meetingsEnvString(MeetingsQuorumListenEnv, m.Quorum.Listen, DefaultMeetingsQuorumListen) +} + +// VexaURL returns the Vexa gateway base URL without a trailing slash. +func (m MeetingsConfig) VexaURL() string { + return strings.TrimRight(meetingsEnvString(MeetingsVexaURLEnv, m.Vexa.URL, ""), "/") +} + +// VexaAPIKeyFile returns the secret file path for the Vexa tenant key. +func (m MeetingsConfig) VexaAPIKeyFile() string { + return meetingsEnvString(MeetingsVexaAPIKeyFileEnv, m.Vexa.APIKeyFile, DefaultMeetingsVexaAPIKeyFile) +} + +// DiscordTokenFile returns the secret file path for the Discord bot token. +func (m MeetingsConfig) DiscordTokenFile() string { + return meetingsEnvString(MeetingsDiscordTokenFileEnv, m.Discord.TokenFile, DefaultMeetingsDiscordTokenFile) +} + +// TranscribeURLOrEmpty returns the transcription service URL. +func (m MeetingsConfig) TranscribeURLOrEmpty() string { + return strings.TrimRight(meetingsEnvString(MeetingsTranscribeURLEnv, m.TranscribeURL, ""), "/") +} + +// DeliveryOrDefault returns the default delivery channel. +func (m MeetingsConfig) DeliveryOrDefault() string { + return strings.ToLower(meetingsEnvString(MeetingsDeliveryEnv, m.Delivery, DefaultMeetingsDelivery)) +} + +// VoiceOnAddressEnabled reports whether hive may speak when addressed by +// name in a meeting that has voice enabled. Default true. +func (m MeetingsConfig) VoiceOnAddressEnabled() bool { + if v, ok := parseBoolEnv(MeetingsVoiceOnAddressEnv); ok { + return v + } + return m.VoiceOnAddress == nil || *m.VoiceOnAddress +} + +// BotNameOrDefault returns the bot display name. +func (m MeetingsConfig) BotNameOrDefault() string { + return meetingsEnvString(MeetingsBotNameEnv, m.BotName, DefaultMeetingsBotName) +} + +// InvitePolicyOrDefault returns who may invite hive to a meeting. +func (m MeetingsConfig) InvitePolicyOrDefault() string { + return strings.ToLower(meetingsEnvString(MeetingsInvitePolicyEnv, m.InvitePolicy, DefaultMeetingsInvitePolicy)) +} + +// ResearchPoolSizeOrDefault returns the research agent pool size. +func (m MeetingsConfig) ResearchPoolSizeOrDefault() int { + return meetingsEnvInt(MeetingsResearchPoolSizeEnv, m.ResearchPoolSize, DefaultMeetingsResearchPoolSize) +} + +// ResearchPerMeetingOrDefault returns the per-meeting research cap. +func (m MeetingsConfig) ResearchPerMeetingOrDefault() int { + return meetingsEnvInt(MeetingsResearchPerMeetingEnv, m.ResearchPerMeeting, DefaultMeetingsResearchPerMeeting) +} + +// ChatPostsPerMinuteOrDefault returns the per-meeting chat post rate. +func (m MeetingsConfig) ChatPostsPerMinuteOrDefault() int { + return meetingsEnvInt(MeetingsChatPostsPerMinuteEnv, m.ChatPostsPerMinute, DefaultMeetingsChatPostsPerMinute) +} + +// LobbyTimeoutSOrDefault returns the lobby timeout in seconds. +func (m MeetingsConfig) LobbyTimeoutSOrDefault() int { + return meetingsEnvInt(MeetingsLobbyTimeoutEnv, m.LobbyTimeoutS, DefaultMeetingsLobbyTimeoutS) +} + +// RetentionDaysOrDefault returns the transcript retention in days. +func (m MeetingsConfig) RetentionDaysOrDefault() int { + return meetingsEnvInt(MeetingsRetentionDaysEnv, m.RetentionDays, DefaultMeetingsRetentionDays) +} + +// ConsentTextOrDefault returns the consent notice. +func (m MeetingsConfig) ConsentTextOrDefault() string { + return meetingsEnvString(MeetingsConsentTextEnv, m.ConsentText, DefaultMeetingsConsentText) +} + +// DataDirOrDefault returns the meetings data directory. +func (m MeetingsConfig) DataDirOrDefault() string { + return meetingsEnvString(MeetingsDataDirEnv, m.DataDir, DefaultMeetingsDataDir) +} + +// MaxTokensPerMeetingOrDefault returns the per-meeting token budget. +func (b MeetingsBudgetConfig) MaxTokensPerMeetingOrDefault() int { + return meetingsEnvInt(MeetingsTokensPerMeetingEnv, b.MaxTokensPerMeeting, DefaultMeetingsMaxTokensPerMeeting) +} + +// MaxTokensPerHourOrDefault returns the hive-wide hourly token ceiling. +func (b MeetingsBudgetConfig) MaxTokensPerHourOrDefault() int { + return meetingsEnvInt(MeetingsTokensPerHourEnv, b.MaxTokensPerHour, DefaultMeetingsMaxTokensPerHour) +} + +// MaxTokensPerDayOrDefault returns the hive-wide daily token ceiling. +func (b MeetingsBudgetConfig) MaxTokensPerDayOrDefault() int { + return meetingsEnvInt(MeetingsTokensPerDayEnv, b.MaxTokensPerDay, DefaultMeetingsMaxTokensPerDay) +} + +// MaxIntentsPerMeetingOrDefault returns the per-meeting intent budget used +// when Spek reports no token usage. +func (b MeetingsBudgetConfig) MaxIntentsPerMeetingOrDefault() int { + return meetingsEnvInt(MeetingsIntentsPerMeetingEnv, b.MaxIntentsPerMeeting, DefaultMeetingsMaxIntentsPerMeeting) +} + +// meetingsSecret resolves a secret: the env value when set, else the +// keystore-sealed file at path. A missing file is not an error; it returns +// "". +func meetingsSecret(ctx context.Context, env, path string) (string, error) { + if v := strings.TrimSpace(os.Getenv(env)); v != "" { + return v, nil + } + v, err := keystore.ReadString(ctx, path) + if errors.Is(err, os.ErrNotExist) { + return "", nil + } + if err != nil { + return "", fmt.Errorf("meetings: reading secret file %s: %w", path, err) + } + return strings.TrimSpace(v), nil +} + +// VexaAPIKey returns the Vexa tenant key: HIVE_MEETINGS_VEXA_API_KEY, then +// the secret file. A secret; never log it. +func (m MeetingsConfig) VexaAPIKey(ctx context.Context) (string, error) { + return meetingsSecret(ctx, MeetingsVexaAPIKeyEnv, m.VexaAPIKeyFile()) +} + +// DiscordToken returns the optional Discord bot token: +// HIVE_MEETINGS_DISCORD_TOKEN, then the secret file. A secret; never log it. +func (m MeetingsConfig) DiscordToken(ctx context.Context) (string, error) { + return meetingsSecret(ctx, MeetingsDiscordTokenEnv, m.DiscordTokenFile()) +} + +// WriteMeetingsSecret seals value into the secret file at path through the +// keystore (owner-only mode). An empty value removes the file. +func WriteMeetingsSecret(ctx context.Context, path, value string) error { + value = strings.TrimSpace(value) + if value == "" { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + return fmt.Errorf("meetings: removing secret file %s: %w", path, err) + } + return nil + } + if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { + return fmt.Errorf("meetings: creating secret dir: %w", err) + } + return keystore.WriteFile(ctx, path, []byte(value)) +} + +// Validate checks the effective meetings settings (env included). A +// disabled block is never rejected, so a stray env var cannot stop a hive +// that does not use meetings from booting. +func (m MeetingsConfig) Validate() error { + if !m.IsEnabled() { + return nil + } + if err := validateMeetingsListen(m.QuorumListen()); err != nil { + return err + } + if u := m.VexaURL(); u != "" { + if err := validateMeetingsURL("meetings.vexa.url", u); err != nil { + return err + } + } + if u := m.TranscribeURLOrEmpty(); u != "" { + if err := validateMeetingsURL("meetings.transcribe_url", u); err != nil { + return err + } + } + if d := m.DeliveryOrDefault(); d != MeetingsDeliveryChat { + return fmt.Errorf("meetings.delivery %q is not supported (only %q)", d, MeetingsDeliveryChat) + } + if p := m.InvitePolicyOrDefault(); p != MeetingsInvitePolicyOwner { + return fmt.Errorf("meetings.invite_policy %q is not supported (only %q)", p, MeetingsInvitePolicyOwner) + } + if n := len([]rune(m.BotNameOrDefault())); n > meetingsMaxBotNameChars { + return fmt.Errorf("meetings.bot_name is %d characters, the limit is %d", n, meetingsMaxBotNameChars) + } + if n := len([]rune(m.ConsentTextOrDefault())); n > meetingsMaxConsentChars { + return fmt.Errorf("meetings.consent_text is %d characters, the limit is %d (one chat post)", n, meetingsMaxConsentChars) + } + for _, f := range []struct { + key string + v int + }{ + {"meetings.research_pool_size", m.ResearchPoolSize}, + {"meetings.research_per_meeting", m.ResearchPerMeeting}, + {"meetings.chat_posts_per_minute", m.ChatPostsPerMinute}, + {"meetings.lobby_timeout_s", m.LobbyTimeoutS}, + {"meetings.retention_days", m.RetentionDays}, + {"meetings.budget.max_tokens_per_meeting", m.Budget.MaxTokensPerMeeting}, + {"meetings.budget.max_tokens_per_hour", m.Budget.MaxTokensPerHour}, + {"meetings.budget.max_tokens_per_day", m.Budget.MaxTokensPerDay}, + {"meetings.budget.max_intents_per_meeting", m.Budget.MaxIntentsPerMeeting}, + } { + if f.v < 0 { + return fmt.Errorf("%s must not be negative (got %d)", f.key, f.v) + } + } + b := m.Budget + if b.MaxTokensPerMeetingOrDefault() > b.MaxTokensPerDayOrDefault() { + return fmt.Errorf("meetings.budget.max_tokens_per_meeting (%d) exceeds max_tokens_per_day (%d)", + b.MaxTokensPerMeetingOrDefault(), b.MaxTokensPerDayOrDefault()) + } + if b.MaxTokensPerHourOrDefault() > b.MaxTokensPerDayOrDefault() { + return fmt.Errorf("meetings.budget.max_tokens_per_hour (%d) exceeds max_tokens_per_day (%d)", + b.MaxTokensPerHourOrDefault(), b.MaxTokensPerDayOrDefault()) + } + return nil +} + +// validateMeetingsListen requires a loopback host:port: Quorum is never +// reachable from outside the container. +func validateMeetingsListen(addr string) error { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return fmt.Errorf("meetings.quorum.listen %q is not host:port: %w", addr, err) + } + if p, err := strconv.Atoi(port); err != nil || p <= 0 || p > 65535 { + return fmt.Errorf("meetings.quorum.listen %q has an invalid port", addr) + } + if host == "localhost" { + return nil + } + if ip := net.ParseIP(host); ip == nil || !ip.IsLoopback() { + return fmt.Errorf("meetings.quorum.listen %q must be a loopback address", addr) + } + return nil +} + +// validateMeetingsURL requires an absolute http(s) URL with no credentials. +func validateMeetingsURL(key, raw string) error { + u, err := url.Parse(raw) + if err != nil { + return fmt.Errorf("%s %q is not a URL: %w", key, raw, err) + } + if (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" { + return fmt.Errorf("%s %q must be an absolute http(s) URL", key, raw) + } + if u.User != nil { + return fmt.Errorf("%s must not carry credentials", key) + } + return nil +} diff --git a/src/pkg/config/meetings_config_test.go b/src/pkg/config/meetings_config_test.go new file mode 100644 index 0000000000..269d2be7a1 --- /dev/null +++ b/src/pkg/config/meetings_config_test.go @@ -0,0 +1,308 @@ +package config + +import ( + "context" + "os" + "path/filepath" + "strings" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestMeetingsConfigDefaults(t *testing.T) { + var m MeetingsConfig + if m.IsEnabled() { + t.Fatal("meetings must default off") + } + checks := []struct { + name string + got, want any + }{ + {"binary", m.QuorumBinary(), DefaultMeetingsQuorumBinary}, + {"listen", m.QuorumListen(), DefaultMeetingsQuorumListen}, + {"vexa", m.VexaURL(), ""}, + {"vexa key file", m.VexaAPIKeyFile(), DefaultMeetingsVexaAPIKeyFile}, + {"discord file", m.DiscordTokenFile(), DefaultMeetingsDiscordTokenFile}, + {"transcribe", m.TranscribeURLOrEmpty(), ""}, + {"delivery", m.DeliveryOrDefault(), MeetingsDeliveryChat}, + {"voice", m.VoiceOnAddressEnabled(), true}, + {"bot", m.BotNameOrDefault(), "hive (notes)"}, + {"invite", m.InvitePolicyOrDefault(), MeetingsInvitePolicyOwner}, + {"pool", m.ResearchPoolSizeOrDefault(), DefaultMeetingsResearchPoolSize}, + {"research cap", m.ResearchPerMeetingOrDefault(), DefaultMeetingsResearchPerMeeting}, + {"chat rate", m.ChatPostsPerMinuteOrDefault(), DefaultMeetingsChatPostsPerMinute}, + {"lobby", m.LobbyTimeoutSOrDefault(), DefaultMeetingsLobbyTimeoutS}, + {"retention", m.RetentionDaysOrDefault(), 30}, + {"consent", m.ConsentTextOrDefault(), DefaultMeetingsConsentText}, + {"data", m.DataDirOrDefault(), DefaultMeetingsDataDir}, + {"tokens meeting", m.Budget.MaxTokensPerMeetingOrDefault(), DefaultMeetingsMaxTokensPerMeeting}, + {"tokens hour", m.Budget.MaxTokensPerHourOrDefault(), DefaultMeetingsMaxTokensPerHour}, + {"tokens day", m.Budget.MaxTokensPerDayOrDefault(), DefaultMeetingsMaxTokensPerDay}, + {"intents", m.Budget.MaxIntentsPerMeetingOrDefault(), DefaultMeetingsMaxIntentsPerMeeting}, + } + for _, c := range checks { + if c.got != c.want { + t.Errorf("%s = %v, want %v", c.name, c.got, c.want) + } + } + if !strings.Contains(DefaultMeetingsConsentText, "hive leave") { + t.Error("the consent notice must tell people how to remove hive") + } +} + +func TestMeetingsConfigYAMLValues(t *testing.T) { + voice := false + m := MeetingsConfig{ + Enabled: true, + Quorum: MeetingsQuorumConfig{Binary: "/opt/q.mjs", Listen: "127.0.0.1:9000"}, + Vexa: MeetingsVexaConfig{URL: "https://vexa.example/", APIKeyFile: "/data/secrets/vk"}, + Discord: MeetingsDiscordConfig{TokenFile: "/data/secrets/dt"}, + TranscribeURL: "http://whisper.local/", + Delivery: "CHAT", + VoiceOnAddress: &voice, + BotName: "notes bot", + InvitePolicy: "Owner", + ResearchPoolSize: 3, + ResearchPerMeeting: 7, + ChatPostsPerMinute: 2, + LobbyTimeoutS: 60, + RetentionDays: 9, + ConsentText: "hello", + DataDir: "/tmp/m", + Budget: MeetingsBudgetConfig{MaxTokensPerMeeting: 10, MaxTokensPerHour: 20, MaxTokensPerDay: 30, MaxIntentsPerMeeting: 40}, + } + if !m.IsEnabled() || m.QuorumBinary() != "/opt/q.mjs" || m.QuorumListen() != "127.0.0.1:9000" { + t.Fatalf("quorum values not honoured: %+v", m.Quorum) + } + if m.VexaURL() != "https://vexa.example" || m.VexaAPIKeyFile() != "/data/secrets/vk" || m.DiscordTokenFile() != "/data/secrets/dt" { + t.Fatal("vexa/discord values not honoured") + } + if m.TranscribeURLOrEmpty() != "http://whisper.local" || m.DeliveryOrDefault() != "chat" || m.VoiceOnAddressEnabled() { + t.Fatal("transcribe/delivery/voice not honoured") + } + if m.BotNameOrDefault() != "notes bot" || m.InvitePolicyOrDefault() != "owner" || m.ConsentTextOrDefault() != "hello" || m.DataDirOrDefault() != "/tmp/m" { + t.Fatal("string values not honoured") + } + if m.ResearchPoolSizeOrDefault() != 3 || m.ResearchPerMeetingOrDefault() != 7 || m.ChatPostsPerMinuteOrDefault() != 2 || + m.LobbyTimeoutSOrDefault() != 60 || m.RetentionDaysOrDefault() != 9 { + t.Fatal("int values not honoured") + } + b := m.Budget + if b.MaxTokensPerMeetingOrDefault() != 10 || b.MaxTokensPerHourOrDefault() != 20 || b.MaxTokensPerDayOrDefault() != 30 || b.MaxIntentsPerMeetingOrDefault() != 40 { + t.Fatal("budget values not honoured") + } + if err := m.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } +} + +func TestMeetingsConfigEnvOverrides(t *testing.T) { + env := map[string]string{ + MeetingsEnabledEnv: "true", + MeetingsQuorumBinaryEnv: "/env/q.mjs", + MeetingsQuorumListenEnv: "localhost:7000", + MeetingsVexaURLEnv: "https://env.vexa/", + MeetingsVexaAPIKeyFileEnv: "/env/vk", + MeetingsDiscordTokenFileEnv: "/env/dt", + MeetingsTranscribeURLEnv: "https://env.whisper", + MeetingsDeliveryEnv: "chat", + MeetingsVoiceOnAddressEnv: "off", + MeetingsBotNameEnv: "env bot", + MeetingsInvitePolicyEnv: "owner", + MeetingsResearchPoolSizeEnv: "4", + MeetingsResearchPerMeetingEnv: "8", + MeetingsChatPostsPerMinuteEnv: "3", + MeetingsLobbyTimeoutEnv: "30", + MeetingsRetentionDaysEnv: "5", + MeetingsConsentTextEnv: "env consent", + MeetingsDataDirEnv: "/env/data", + MeetingsTokensPerMeetingEnv: "11", + MeetingsTokensPerHourEnv: "12", + MeetingsTokensPerDayEnv: "13", + MeetingsIntentsPerMeetingEnv: "14", + } + for k, v := range env { + t.Setenv(k, v) + } + m := MeetingsConfig{ResearchPoolSize: 9, BotName: "yaml bot", Budget: MeetingsBudgetConfig{MaxTokensPerDay: 99}} + if !m.IsEnabled() || m.QuorumBinary() != "/env/q.mjs" || m.QuorumListen() != "localhost:7000" || + m.VexaURL() != "https://env.vexa" || m.VexaAPIKeyFile() != "/env/vk" || m.DiscordTokenFile() != "/env/dt" || + m.TranscribeURLOrEmpty() != "https://env.whisper" || m.VoiceOnAddressEnabled() || m.BotNameOrDefault() != "env bot" || + m.ConsentTextOrDefault() != "env consent" || m.DataDirOrDefault() != "/env/data" { + t.Fatal("string env overrides not honoured") + } + if m.ResearchPoolSizeOrDefault() != 4 || m.ResearchPerMeetingOrDefault() != 8 || m.ChatPostsPerMinuteOrDefault() != 3 || + m.LobbyTimeoutSOrDefault() != 30 || m.RetentionDaysOrDefault() != 5 { + t.Fatal("int env overrides not honoured") + } + b := m.Budget + if b.MaxTokensPerMeetingOrDefault() != 11 || b.MaxTokensPerHourOrDefault() != 12 || b.MaxTokensPerDayOrDefault() != 13 || b.MaxIntentsPerMeetingOrDefault() != 14 { + t.Fatal("budget env overrides not honoured") + } + if err := m.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + + t.Setenv(MeetingsEnabledEnv, "0") + if (MeetingsConfig{Enabled: true}).IsEnabled() { + t.Fatal("HIVE_MEETINGS_ENABLED=0 must win over meetings.enabled") + } +} + +func TestMeetingsConfigInvalidIntEnvFallsBack(t *testing.T) { + t.Setenv(MeetingsResearchPoolSizeEnv, "lots") + t.Setenv(MeetingsRetentionDaysEnv, "-3") + m := MeetingsConfig{ResearchPoolSize: 6} + if got := m.ResearchPoolSizeOrDefault(); got != 6 { + t.Fatalf("pool = %d, want the yaml value 6", got) + } + if got := m.RetentionDaysOrDefault(); got != DefaultMeetingsRetentionDays { + t.Fatalf("retention = %d, want the default", got) + } +} + +func TestMeetingsConfigValidate(t *testing.T) { + long := strings.Repeat("x", meetingsMaxConsentChars+1) + cases := []struct { + name string + m MeetingsConfig + want string + }{ + {"non-loopback listen", MeetingsConfig{Quorum: MeetingsQuorumConfig{Listen: "0.0.0.0:8765"}}, "loopback"}, + {"hostname listen", MeetingsConfig{Quorum: MeetingsQuorumConfig{Listen: "quorum:8765"}}, "loopback"}, + {"no port", MeetingsConfig{Quorum: MeetingsQuorumConfig{Listen: "127.0.0.1"}}, "host:port"}, + {"bad port", MeetingsConfig{Quorum: MeetingsQuorumConfig{Listen: "127.0.0.1:99999"}}, "invalid port"}, + {"vexa scheme", MeetingsConfig{Vexa: MeetingsVexaConfig{URL: "ftp://vexa"}}, "http(s)"}, + {"vexa parse", MeetingsConfig{Vexa: MeetingsVexaConfig{URL: "http://[::1"}}, "not a URL"}, + {"vexa creds", MeetingsConfig{Vexa: MeetingsVexaConfig{URL: "https://u:p@vexa"}}, "credentials"}, + {"transcribe", MeetingsConfig{TranscribeURL: "whisper"}, "transcribe_url"}, + {"delivery", MeetingsConfig{Delivery: "email"}, "delivery"}, + {"invite", MeetingsConfig{InvitePolicy: "anyone"}, "invite_policy"}, + {"bot name", MeetingsConfig{BotName: strings.Repeat("b", meetingsMaxBotNameChars+1)}, "bot_name"}, + {"consent", MeetingsConfig{ConsentText: long}, "consent_text"}, + {"negative", MeetingsConfig{RetentionDays: -1}, "retention_days"}, + {"meeting over day", MeetingsConfig{Budget: MeetingsBudgetConfig{MaxTokensPerMeeting: 5, MaxTokensPerDay: 4, MaxTokensPerHour: 1}}, "max_tokens_per_meeting"}, + {"hour over day", MeetingsConfig{Budget: MeetingsBudgetConfig{MaxTokensPerMeeting: 1, MaxTokensPerDay: 4, MaxTokensPerHour: 5}}, "max_tokens_per_hour"}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + c.m.Enabled = true + err := c.m.Validate() + if err == nil || !strings.Contains(err.Error(), c.want) { + t.Fatalf("Validate = %v, want an error containing %q", err, c.want) + } + c.m.Enabled = false + if err := c.m.Validate(); err != nil { + t.Fatalf("a disabled block must validate, got %v", err) + } + }) + } + t.Run("ipv6 loopback", func(t *testing.T) { + m := MeetingsConfig{Enabled: true, Quorum: MeetingsQuorumConfig{Listen: "[::1]:8765"}} + if err := m.Validate(); err != nil { + t.Fatalf("Validate: %v", err) + } + }) +} + +func TestMeetingsConfigYAMLKeys(t *testing.T) { + src := ` +enabled: true +quorum: {binary: /q.mjs, listen: "127.0.0.1:1"} +vexa: {url: "https://v", api_key_file: /data/secrets/k} +discord: {token_file: /data/secrets/d} +transcribe_url: https://w +delivery: chat +voice_on_address: false +bot_name: b +invite_policy: owner +research_pool_size: 2 +research_per_meeting: 3 +chat_posts_per_minute: 4 +lobby_timeout_s: 5 +budget: {max_tokens_per_meeting: 6, max_tokens_per_hour: 7, max_tokens_per_day: 8, max_intents_per_meeting: 9} +retention_days: 10 +consent_text: c +data_dir: /d +` + var m MeetingsConfig + if err := yaml.Unmarshal([]byte(src), &m); err != nil { + t.Fatal(err) + } + if !m.Enabled || m.Quorum.Listen != "127.0.0.1:1" || m.Vexa.APIKeyFile != "/data/secrets/k" || m.Discord.TokenFile != "/data/secrets/d" || + m.VoiceOnAddress == nil || *m.VoiceOnAddress || m.Budget.MaxIntentsPerMeeting != 9 || m.DataDir != "/d" || m.LobbyTimeoutS != 5 { + t.Fatalf("yaml keys not decoded: %+v", m) + } +} + +func TestMeetingsSecrets(t *testing.T) { + dir := t.TempDir() + ctx := context.Background() + key := filepath.Join(dir, "secrets", "vexa") + tok := filepath.Join(dir, "secrets", "discord") + m := MeetingsConfig{Vexa: MeetingsVexaConfig{APIKeyFile: key}, Discord: MeetingsDiscordConfig{TokenFile: tok}} + + if v, err := m.VexaAPIKey(ctx); err != nil || v != "" { + t.Fatalf("missing file: %q, %v; want empty and no error", v, err) + } + if err := WriteMeetingsSecret(ctx, key, " tenant-key \n"); err != nil { + t.Fatal(err) + } + if fi, err := os.Stat(key); err != nil || fi.Mode().Perm() != 0o600 { + t.Fatalf("secret file mode: %v, %v", fi, err) + } + if v, err := m.VexaAPIKey(ctx); err != nil || v != "tenant-key" { + t.Fatalf("VexaAPIKey = %q, %v", v, err) + } + if err := WriteMeetingsSecret(ctx, tok, "bot-token"); err != nil { + t.Fatal(err) + } + if v, err := m.DiscordToken(ctx); err != nil || v != "bot-token" { + t.Fatalf("DiscordToken = %q, %v", v, err) + } + + t.Setenv(MeetingsVexaAPIKeyEnv, "from-env") + t.Setenv(MeetingsDiscordTokenEnv, "tok-env") + if v, _ := m.VexaAPIKey(ctx); v != "from-env" { + t.Fatalf("env key must win, got %q", v) + } + if v, _ := m.DiscordToken(ctx); v != "tok-env" { + t.Fatalf("env token must win, got %q", v) + } + t.Setenv(MeetingsVexaAPIKeyEnv, "") + + // Clearing removes the file; clearing twice is fine. + if err := WriteMeetingsSecret(ctx, key, ""); err != nil { + t.Fatal(err) + } + if err := WriteMeetingsSecret(ctx, key, ""); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(key); !os.IsNotExist(err) { + t.Fatalf("cleared secret still present: %v", err) + } + + // A directory where the file should be is a read error, not "unset". + if err := os.MkdirAll(key, 0o700); err != nil { + t.Fatal(err) + } + if _, err := m.VexaAPIKey(ctx); err == nil { + t.Fatal("want an error reading a directory as a secret") + } + // Removing a non-empty directory fails. + if err := os.WriteFile(filepath.Join(key, "x"), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if err := WriteMeetingsSecret(ctx, key, ""); err == nil { + t.Fatal("want an error removing a non-empty directory") + } + // The parent cannot be created under a regular file. + blocker := filepath.Join(dir, "file") + if err := os.WriteFile(blocker, []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + if err := WriteMeetingsSecret(ctx, filepath.Join(blocker, "s"), "v"); err == nil { + t.Fatal("want an error when the secret dir cannot be created") + } +} diff --git a/src/pkg/config/testdata/agent_runtime_golden.json b/src/pkg/config/testdata/agent_runtime_golden.json index e2bb831d7a..80d7c44248 100644 --- a/src/pkg/config/testdata/agent_runtime_golden.json +++ b/src/pkg/config/testdata/agent_runtime_golden.json @@ -784,6 +784,12 @@ } }, "retro": {}, + "meetings": { + "quorum": {}, + "vexa": {}, + "discord": {}, + "budget": {} + }, "swarm": {}, "autonomy": {}, "review": { diff --git a/src/pkg/config/validate.go b/src/pkg/config/validate.go index 5292384e73..7d559c1c7a 100644 --- a/src/pkg/config/validate.go +++ b/src/pkg/config/validate.go @@ -222,6 +222,9 @@ func (c *Config) ValidateWithOptions(opts ValidateOptions) error { if err := c.Hub.ValidateNPS(); err != nil { return err } + if err := c.Meetings.Validate(); err != nil { + return err + } if err := c.Review.ValidateFixerFallback(); err != nil { return err } diff --git a/src/pkg/dashboard/api_sessions_keyboard_test.go b/src/pkg/dashboard/api_sessions_keyboard_test.go index 8b9acbce0a..653c2f26a0 100644 --- a/src/pkg/dashboard/api_sessions_keyboard_test.go +++ b/src/pkg/dashboard/api_sessions_keyboard_test.go @@ -5,6 +5,7 @@ import ( "net/http/httptest" "net/url" "strings" + "sync" "testing" "time" @@ -420,16 +421,62 @@ func TestSessionsSendIsQueuedUntilThePaneIsIdle(t *testing.T) { } } +// fakeQueueClock is a hand-advanced clock for the queue worker's fail-open +// deadline, so the test never depends on how fast the machine runs. +type fakeQueueClock struct { + mu sync.Mutex + cur time.Time +} + +func (c *fakeQueueClock) Now() time.Time { + c.mu.Lock() + defer c.mu.Unlock() + return c.cur +} + +func (c *fakeQueueClock) Advance(d time.Duration) { + c.mu.Lock() + defer c.mu.Unlock() + c.cur = c.cur.Add(d) +} + +// waitPolls waits until the pane has been read n more times, which gives a +// "nothing else happened" assertion real polls to be wrong in, without a sleep. +func (e *sessionTestEnv) waitPolls(t *testing.T, n int) { + t.Helper() + target := e.rt.captureCount() + n + waitSessionsFor(t, "more pane polls", func() bool { return e.rt.captureCount() >= target }) +} + func TestSessionsSendFailsOpenAfterTheDeadline(t *testing.T) { + const deadline = time.Minute e := newSessionTestEnv(t, true) fastSessionQueue(t, e) - t.Setenv(sessionQueueDeliverAfterEnv, "1") + clock := &fakeQueueClock{cur: time.Unix(1_700_000_000, 0)} + e.s.sessionQueueTiming.now = clock.Now + e.s.sessionQueueTiming.deliverAfter = deadline id := e.create(t, callerRW, "stuck") e.rt.setCapture(claudeBusyPane) if got := e.statusOf(t, callerRW, "POST", "/api/sessions/"+id+"/input", map[string]any{"text": "eventually"}); got != http.StatusAccepted { t.Fatalf("input = %d", got) } + // The worker is polling the busy pane; before the deadline nothing is typed. + waitSessionsFor(t, "the worker to pick up the head", func() bool { + return sessionOf(t, e, callerRW, id)["queue"].(map[string]any)["items"].([]any)[0].(map[string]any)["delivering"] == true + }) + clock.Advance(deadline - time.Second) + e.waitPolls(t, 10) + if typed := e.rt.typedTexts(); len(typed) != 0 { + t.Fatalf("delivered before the deadline: %v", typed) + } + // Past the deadline the send fails open, and is delivered exactly once. + clock.Advance(2 * time.Second) waitSessionsFor(t, "fail-open delivery", func() bool { return len(e.rt.typedTexts()) == 1 }) + waitSessionsFor(t, "queue to drain", func() bool { return sessionOf(t, e, callerRW, id)["queue"].(map[string]any)["length"] == float64(0) }) + // The worker has exited with the queue empty, so nothing can type again. + if typed := e.rt.typedTexts(); len(typed) != 1 || typed[0] != "eventually" { + t.Errorf("typed = %v, want exactly one delivery of the queued text", typed) + } } func TestSessionsQueueCancelBoundsAndEndedSession(t *testing.T) { diff --git a/src/pkg/dashboard/api_sessions_queue.go b/src/pkg/dashboard/api_sessions_queue.go index bcc95de264..c6da2a2048 100644 --- a/src/pkg/dashboard/api_sessions_queue.go +++ b/src/pkg/dashboard/api_sessions_queue.go @@ -46,8 +46,27 @@ const ( // sessionQueuePacing is the worker's poll interval and post-delivery pause; // zero values mean the defaults. Set on the Server before any worker starts. +// now, when set, replaces the wall clock the fail-open deadline is measured +// against, and deliverAfter, when positive, replaces the env/default deadline +// (both are test seams, so a test advances a fake clock instead of sleeping). type sessionQueuePacing struct { poll, postDeliver time.Duration + deliverAfter time.Duration + now func() time.Time +} + +func (s *Server) sessionQueueNow() time.Time { + if s.sessionQueueTiming.now != nil { + return s.sessionQueueTiming.now() + } + return time.Now() +} + +func (s *Server) sessionQueueDeadline() time.Duration { + if s.sessionQueueTiming.deliverAfter > 0 { + return s.sessionQueueTiming.deliverAfter + } + return sessionQueueDeliverAfter() } func (s *Server) sessionQueuePoll() time.Duration { @@ -361,14 +380,14 @@ func (s *Server) sessionQueueHeadIs(id, itemID string) bool { // waitSessionIdle blocks until the pane looks idle, the fail-open deadline // passes or the item is gone from the queue, and returns how long it waited. func (s *Server) waitSessionIdle(rt SessionRuntime, id, vendor, itemID string) time.Duration { - start := time.Now() - deadline := start.Add(sessionQueueDeliverAfter()) + start := s.sessionQueueNow() + deadline := start.Add(s.sessionQueueDeadline()) for { if text, err := rt.Capture(id, sessionIdleCaptureLines); err == nil && sessionPaneIdle(vendor, text) { - return time.Since(start) + return s.sessionQueueNow().Sub(start) } - if !time.Now().Before(deadline) || !s.sessionQueueHeadIs(id, itemID) { - return time.Since(start) + if !s.sessionQueueNow().Before(deadline) || !s.sessionQueueHeadIs(id, itemID) { + return s.sessionQueueNow().Sub(start) } time.Sleep(s.sessionQueuePoll()) } diff --git a/src/pkg/dashboard/api_sessions_test.go b/src/pkg/dashboard/api_sessions_test.go index 69b25e2041..ea8423eb81 100644 --- a/src/pkg/dashboard/api_sessions_test.go +++ b/src/pkg/dashboard/api_sessions_test.go @@ -23,18 +23,19 @@ import ( // fakeSessionRuntime is a deterministic SessionRuntime: no tmux, no processes. type fakeSessionRuntime struct { - mu sync.Mutex - vendors []crux.UserSessionVendor - workDir string - alive map[string]crux.UserSessionProbe - started []crux.UserSessionSpec - killed []string - typed []string - submits []bool - capture string - resume map[string]crux.UserSessionResume - startFn func(crux.UserSessionSpec) error - removed []string + mu sync.Mutex + vendors []crux.UserSessionVendor + workDir string + alive map[string]crux.UserSessionProbe + started []crux.UserSessionSpec + killed []string + typed []string + submits []bool + capture string + captures int // times Capture was called + resume map[string]crux.UserSessionResume + startFn func(crux.UserSessionSpec) error + removed []string // removeErr, when set, makes Remove fail (the files could not be deleted). removeErr error // resumeDirs records the workDir each ResumeInfo call was given. @@ -93,9 +94,17 @@ func (f *fakeSessionRuntime) Kill(id string) error { func (f *fakeSessionRuntime) Capture(id string, lines int) (string, error) { f.mu.Lock() defer f.mu.Unlock() + f.captures++ return f.capture, nil } +// captureCount is how many times the pane has been read so far. +func (f *fakeSessionRuntime) captureCount() int { + f.mu.Lock() + defer f.mu.Unlock() + return f.captures +} + // setCapture replaces what Capture returns (the pane's last lines). func (f *fakeSessionRuntime) setCapture(text string) { f.mu.Lock() diff --git a/src/pkg/dashboard/inbox_transcribe.test.js b/src/pkg/dashboard/inbox_transcribe.test.js index ff4385fd22..3af2074df8 100644 --- a/src/pkg/dashboard/inbox_transcribe.test.js +++ b/src/pkg/dashboard/inbox_transcribe.test.js @@ -116,7 +116,7 @@ const formRoot = fields => ({ }); const settingsHarness = () => new Function('escapeHtml', 'yamlHint', 'INBOX_TX_BYTES_PER_MB', - fnSrc('inboxTranscriptionFormHtml') + '\n' + fnSrc('inboxReadTranscription') + '\nreturn { inboxTranscriptionFormHtml, inboxReadTranscription };', + fnSrc('inboxTranscriptionFormHtml') + '\n' + fnSrc('inboxTranscriptionUsageHtml') + '\n' + fnSrc('inboxReadTranscription') + '\nreturn { inboxTranscriptionFormHtml, inboxTranscriptionUsageHtml, inboxReadTranscription };', )(escapeHtml, p => '' + p + '', BYTES_PER_MB); test('settings section shows the consent notice, key status and every yaml key', () => { @@ -159,3 +159,19 @@ test('reading the section enforces consent and converts the size limit', () => { assert.strictEqual(inboxReadTranscription(null), null); assert.strictEqual(inboxReadTranscription({ querySelector: () => null }), null, 'no section, no transcription payload'); }); + +test('settings section shows month usage, egress and the auto-mode source hint', () => { + const { inboxTranscriptionFormHtml, inboxTranscriptionUsageHtml } = settingsHarness(); + let out = inboxTranscriptionUsageHtml({ month_minutes_used: 123.456, monthly_budget_minutes: 600, today_items: 3, daily_max_items: 20, egress_hosts: ['api.deepgram.com'] }); + assert.match(out, /data-status="ok">123\.5 of 600 minutes used/); + assert.match(out, /today: 3 of 20 recordings/); + assert.match(out, /api\.deepgram\.com:443<\/code>/); + assert.doesNotMatch(out, /none are listed/); + out = inboxTranscriptionUsageHtml({ month_minutes_used: 590, monthly_budget_minutes: 600, mode: 'auto', sources: [] }); + assert.match(out, /data-status="syncing">590 of 600/); + assert.match(out, /none are listed/); + assert.doesNotMatch(out, /Egress/); + assert.match(inboxTranscriptionUsageHtml({ month_minutes_used: 700, monthly_budget_minutes: 600 }), /data-status="error"/); + assert.match(inboxTranscriptionUsageHtml(undefined), /0 of 0 minutes used/); + assert.match(inboxTranscriptionFormHtml({ month_minutes_used: 1, monthly_budget_minutes: 600 }), /data-inbox-tx-usage="1"/); +}); diff --git a/src/pkg/dashboard/settings_parity_allowlist.txt b/src/pkg/dashboard/settings_parity_allowlist.txt index 79c14423aa..17550c6abe 100644 --- a/src/pkg/dashboard/settings_parity_allowlist.txt +++ b/src/pkg/dashboard/settings_parity_allowlist.txt @@ -440,6 +440,28 @@ knowledge.vaults[].auto_index # TODO #11854 knowledge.vaults[].git_sync # TODO #11854 knowledge.vaults[].name # TODO #11854 knowledge.vaults[].path # TODO #11854 +meetings.enabled # TODO #18292 (Settings, Meetings panel) +meetings.quorum.binary # TODO #18292 (Settings, Meetings panel) +meetings.quorum.listen # TODO #18292 (Settings, Meetings panel) +meetings.vexa.url # TODO #18292 (Settings, Meetings panel) +meetings.vexa.api_key_file # TODO #18292 (Settings, Meetings panel) +meetings.discord.token_file # TODO #18292 (Settings, Meetings panel) +meetings.transcribe_url # TODO #18292 (Settings, Meetings panel) +meetings.delivery # TODO #18292 (Settings, Meetings panel) +meetings.voice_on_address # TODO #18292 (Settings, Meetings panel) +meetings.bot_name # TODO #18292 (Settings, Meetings panel) +meetings.invite_policy # TODO #18292 (Settings, Meetings panel) +meetings.research_pool_size # TODO #18292 (Settings, Meetings panel) +meetings.research_per_meeting # TODO #18292 (Settings, Meetings panel) +meetings.chat_posts_per_minute # TODO #18292 (Settings, Meetings panel) +meetings.lobby_timeout_s # TODO #18292 (Settings, Meetings panel) +meetings.budget.max_tokens_per_meeting # TODO #18292 (Settings, Meetings panel) +meetings.budget.max_tokens_per_hour # TODO #18292 (Settings, Meetings panel) +meetings.budget.max_tokens_per_day # TODO #18292 (Settings, Meetings panel) +meetings.budget.max_intents_per_meeting # TODO #18292 (Settings, Meetings panel) +meetings.retention_days # TODO #18292 (Settings, Meetings panel) +meetings.consent_text # TODO #18292 (Settings, Meetings panel) +meetings.data_dir # TODO #18292 (Settings, Meetings panel) network.upstream_proxy.hosts[] # deployment-level egress knob (corporate forward proxy), set via hive.yaml/env/hub clusters.json, not a Settings control (#17637) network.upstream_proxy.no_proxy[] # deployment-level egress knob (corporate forward proxy), set via hive.yaml/env/hub clusters.json, not a Settings control (#17637) network.upstream_proxy.url # deployment-level egress knob (corporate forward proxy), set via hive.yaml/env/hub clusters.json, not a Settings control (#17637) diff --git a/src/pkg/dashboard/static-src/js/inbox.js b/src/pkg/dashboard/static-src/js/inbox.js index 21a599bc6c..44e5cb32c8 100644 --- a/src/pkg/dashboard/static-src/js/inbox.js +++ b/src/pkg/dashboard/static-src/js/inbox.js @@ -474,10 +474,26 @@ field('max_mb', 'Largest upload (MB)', sizeMB, '500', 'number') + '' + '' + + inboxTranscriptionUsageHtml(tx) + '

Key $' + escapeHtml(env) + ': ' + (tx.key_present ? 'key present' : 'key missing set it in this hive\'s environment (it is never stored in hive.yaml)') + '. ' + yamlHint('inbox.transcription.mode') + ' ' + yamlHint('inbox.transcription.sources[]') + ' ' + yamlHint('inbox.transcription.api_key_env') + ' ' + yamlHint('inbox.transcription.provider') + ' ' + yamlHint('inbox.transcription.model') + ' ' + yamlHint('inbox.transcription.diarize') + ' ' + yamlHint('inbox.transcription.language') + ' ' + yamlHint('inbox.transcription.max_duration_minutes') + ' ' + yamlHint('inbox.transcription.max_bytes') + ' ' + yamlHint('inbox.transcription.monthly_budget_minutes') + ' ' + yamlHint('inbox.transcription.daily_max_items') + '

'; } + // This month's spend on the owner's own Deepgram account, today's runs, + // the egress the hive needs, and a hint when auto mode has no sources. + function inboxTranscriptionUsageHtml(tx) { + tx = tx || {}; + const used = Math.round((Number(tx.month_minutes_used) || 0) * 10) / 10; + const budget = Number(tx.monthly_budget_minutes) || 0; + const pct = budget > 0 ? Math.min(100, Math.round(used / budget * 100)) : 0; + const state = pct >= 100 ? 'error' : (pct >= 80 ? 'syncing' : 'ok'); + let html = '

This month: ' + used + ' of ' + budget + ' minutes used' + + ' · today: ' + (Number(tx.today_items) || 0) + ' of ' + (Number(tx.daily_max_items) || 0) + ' recordings (UTC).

'; + const hosts = Array.isArray(tx.egress_hosts) ? tx.egress_hosts : []; + if (hosts.length) html += '

Egress: this hive must reach ' + hosts.map(h => '' + escapeHtml(h) + ':443').join(', ') + ' while transcription is enabled.

'; + if (tx.mode === 'auto' && !(tx.sources || []).length) html += '

Auto mode transcribes only the sources listed above; none are listed, so nothing is transcribed automatically.

'; + return html; + } // inboxReadTranscription reads the section back into inbox.transcription. // It returns { error } when enabling without the consent box. function inboxReadTranscription(root, current) { diff --git a/src/pkg/dashboard/static/index.html b/src/pkg/dashboard/static/index.html index 7acae5d5c1..df5b5e98e9 100644 --- a/src/pkg/dashboard/static/index.html +++ b/src/pkg/dashboard/static/index.html @@ -50428,10 +50428,26 @@ field('max_mb', 'Largest upload (MB)', sizeMB, '500', 'number') + '' + '' + + inboxTranscriptionUsageHtml(tx) + '

Key $' + escapeHtml(env) + ': ' + (tx.key_present ? 'key present' : 'key missing set it in this hive\'s environment (it is never stored in hive.yaml)') + '. ' + yamlHint('inbox.transcription.mode') + ' ' + yamlHint('inbox.transcription.sources[]') + ' ' + yamlHint('inbox.transcription.api_key_env') + ' ' + yamlHint('inbox.transcription.provider') + ' ' + yamlHint('inbox.transcription.model') + ' ' + yamlHint('inbox.transcription.diarize') + ' ' + yamlHint('inbox.transcription.language') + ' ' + yamlHint('inbox.transcription.max_duration_minutes') + ' ' + yamlHint('inbox.transcription.max_bytes') + ' ' + yamlHint('inbox.transcription.monthly_budget_minutes') + ' ' + yamlHint('inbox.transcription.daily_max_items') + '

'; } + // This month's spend on the owner's own Deepgram account, today's runs, + // the egress the hive needs, and a hint when auto mode has no sources. + function inboxTranscriptionUsageHtml(tx) { + tx = tx || {}; + const used = Math.round((Number(tx.month_minutes_used) || 0) * 10) / 10; + const budget = Number(tx.monthly_budget_minutes) || 0; + const pct = budget > 0 ? Math.min(100, Math.round(used / budget * 100)) : 0; + const state = pct >= 100 ? 'error' : (pct >= 80 ? 'syncing' : 'ok'); + let html = '

This month: ' + used + ' of ' + budget + ' minutes used' + + ' · today: ' + (Number(tx.today_items) || 0) + ' of ' + (Number(tx.daily_max_items) || 0) + ' recordings (UTC).

'; + const hosts = Array.isArray(tx.egress_hosts) ? tx.egress_hosts : []; + if (hosts.length) html += '

Egress: this hive must reach ' + hosts.map(h => '' + escapeHtml(h) + ':443').join(', ') + ' while transcription is enabled.

'; + if (tx.mode === 'auto' && !(tx.sources || []).length) html += '

Auto mode transcribes only the sources listed above; none are listed, so nothing is transcribed automatically.

'; + return html; + } // inboxReadTranscription reads the section back into inbox.transcription. // It returns { error } when enabling without the consent box. function inboxReadTranscription(root, current) { diff --git a/src/pkg/inbox/transcription.go b/src/pkg/inbox/transcription.go index 19ce867e20..5ce42b034a 100644 --- a/src/pkg/inbox/transcription.go +++ b/src/pkg/inbox/transcription.go @@ -60,6 +60,9 @@ type TranscriptionSettings struct { // MonthMinutesUsed / TodayItems are this spoke's recorded spend. MonthMinutesUsed float64 `json:"month_minutes_used"` TodayItems int `json:"today_items"` + // EgressHosts are the hosts the hive must reach while transcription is + // enabled (api.deepgram.com), for egress allowlists. + EgressHosts []string `json:"egress_hosts"` } // TranscribeAudit is one inbox_transcribe audit entry. It never carries the @@ -128,6 +131,7 @@ func (s *Service) TranscriptionSettings() TranscriptionSettings { DailyMaxItems: t.EffectiveDailyMaxItems(), ConsentAcknowledged: t.ConsentAcknowledged, } out.KeyPresent = strings.TrimSpace(s.getenv(out.APIKeyEnv)) != "" + out.EgressHosts = TranscriptionEgressHosts(t) if store := s.transcriptStore(); store != nil { u := store.Usage(s.now()) out.MonthMinutesUsed, out.TodayItems = u.MonthSeconds/time.Minute.Seconds(), u.TodayItems diff --git a/src/pkg/inbox/transcription_sweep.go b/src/pkg/inbox/transcription_sweep.go new file mode 100644 index 0000000000..b962820dff --- /dev/null +++ b/src/pkg/inbox/transcription_sweep.go @@ -0,0 +1,106 @@ +package inbox + +import ( + "context" + "strings" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/transcribe" +) + +const ( + // AutoTranscribeActor is the audit actor of sweep-started runs. + AutoTranscribeActor = "inbox-auto-transcribe" + // autoRetryBase is the wait before the sweep retries a failed item; it + // doubles per attempt (1h, 2h) until TranscriptionMaxAttempts. + autoRetryBase = time.Hour +) + +// TranscribeSweep queues machine transcriptions for pending items from the +// sources listed in inbox.transcription.sources. It runs only in auto mode +// with transcription enabled and consented, and is called after each +// connector sync. Runs still execute one at a time; the sweep only queues +// what today's item cap and this month's minutes leave room for, so a +// backlog never turns into a burst of budget skips. +// +// Retries: a failed item is retried after a backoff (1h, then 2h) until it +// has had TranscriptionMaxAttempts runs; an item skipped for budget is +// retried once the UTC day has turned; too long, too large and unsupported +// skips are never retried automatically (the owner can still click +// Transcribe). +func (s *Service) TranscribeSweep(ctx context.Context, opts TranscribeOptions) (queued int, err error) { + cfg := s.Config().Transcription + store := s.transcriptStore() + if store == nil || !cfg.Active() || cfg.EffectiveMode() != config.InboxTranscriptionModeAuto { + return 0, nil + } + items, err := s.Items(ctx) + now := s.now() + u := store.Usage(now) + itemsLeft := cfg.EffectiveDailyMaxItems() - u.TodayItems + timeLeft := cfg.EffectiveMonthlyBudget() - time.Duration(u.MonthSeconds*float64(time.Second)) + seen := map[string]bool{} + for _, it := range items { + if itemsLeft <= 0 { + break + } + if seen[it.Key()] || !it.Transcribable || !(cfg.AutoSource(it.Source) || cfg.AutoSource(it.Connector)) || !autoDue(it.Transcription, now) { + continue + } + seen[it.Key()] = true + dur := time.Duration(it.DurationMS) * time.Millisecond + if dur > timeLeft { + // Leave it pending for next month instead of recording a skip. + continue + } + st, terr := s.Transcribe(ctx, it, opts) + if terr != nil { + s.Logger.Warn("inbox: auto transcription", "item", it.Key(), "error", terr) + continue + } + if st.Status == TranscriptionQueued { + queued++ + itemsLeft-- + timeLeft -= dur + } + } + return queued, err +} + +// autoDue reports whether the sweep should (re)try an item now. +func autoDue(st *TranscriptionState, now time.Time) bool { + if st == nil || st.Status == "" { + return true + } + if st.Attempts >= TranscriptionMaxAttempts { + return false + } + switch st.Status { + case TranscriptionFailed: + return !now.Before(st.UpdatedAt.Add(transcribe.ExponentialBackoff(autoRetryBase)(st.Attempts))) + case TranscriptionSkipped: + return strings.HasPrefix(st.Reason, SkipBudget) && now.UTC().Format(usageDayLayout) != st.UpdatedAt.UTC().Format(usageDayLayout) + } + return false +} + +// TranscriptionEgressHosts lists the hosts this hive must reach for +// transcription: api.deepgram.com while transcription is enabled, else none. +func TranscriptionEgressHosts(t config.InboxTranscription) []string { + if !t.Enabled { + return []string{} + } + return []string{transcribe.DeepgramHost} +} + +// Detail renders the entry as "key=value, key=value", the audit-log detail +// shape the dashboard writes for request-driven runs. +func (a TranscribeAudit) Detail() string { + kv := a.Fields() + parts := make([]string, 0, len(kv)/2) + for i := 0; i+1 < len(kv); i += 2 { + parts = append(parts, kv[i]+"="+kv[i+1]) + } + return strings.Join(parts, ", ") +} diff --git a/src/pkg/inbox/transcription_sweep_test.go b/src/pkg/inbox/transcription_sweep_test.go new file mode 100644 index 0000000000..003fc487a2 --- /dev/null +++ b/src/pkg/inbox/transcription_sweep_test.go @@ -0,0 +1,179 @@ +package inbox + +import ( + "context" + "errors" + "net/http" + "testing" + "time" + + "github.com/hivecommons/hive/pkg/config" + "github.com/hivecommons/hive/pkg/transcribe" +) + +func autoTx(sources ...string) config.InboxTranscription { + return config.InboxTranscription{Enabled: true, ConsentAcknowledged: true, Mode: config.InboxTranscriptionModeAuto, Sources: sources} +} + +func TestTranscribeSweepOnlyAutoModeOptedInSources(t *testing.T) { + srv := audioServer(t, http.StatusOK, txAudio) + ctx := context.Background() + for name, tx := range map[string]config.InboxTranscription{ + "manual mode": {Enabled: true, ConsentAcknowledged: true, Sources: []string{"plaud"}}, + "not opted in": autoTx("zoom"), + "disabled": {Mode: config.InboxTranscriptionModeAuto, Sources: []string{"plaud"}}, + "no consent yet": {Enabled: true, Mode: config.InboxTranscriptionModeAuto, Sources: []string{"plaud"}}, + } { + e := newTxEnv(t, tx, srv.URL) + if n, err := e.svc.TranscribeSweep(ctx, e.opts()); n != 0 || err != nil || e.fake.Calls() != 0 { + t.Fatalf("%s: queued %d err %v calls %d", name, n, err, e.fake.Calls()) + } + } + noStore := NewService(config.InboxConfig{Transcription: autoTx("plaud")}, NewStore(""), nil, nil, nil) + if n, _ := noStore.TranscribeSweep(ctx, TranscribeOptions{}); n != 0 { + t.Fatal("no store, no sweep") + } + + e := newTxEnv(t, autoTx("plaud"), srv.URL) + n, err := e.svc.TranscribeSweep(ctx, TranscribeOptions{Actor: AutoTranscribeActor, Audit: e.opts().Audit}) + if n != 1 || err != nil || e.fake.Calls() != 1 { + t.Fatalf("auto sweep queued %d err %v calls %d", n, err, e.fake.Calls()) + } + if rec, _ := e.store.Get("plaud", "rec-1"); rec.Status != TranscriptionDone { + t.Fatalf("record = %+v", rec) + } + if len(e.audits) != 1 || e.audits[0].Actor != AutoTranscribeActor { + t.Fatalf("audits = %+v", e.audits) + } + // Dedupe: a second sweep finds nothing to do. + if n, _ := e.svc.TranscribeSweep(ctx, e.opts()); n != 0 || e.fake.Calls() != 1 { + t.Fatalf("second sweep queued %d calls %d", n, e.fake.Calls()) + } + // A connector name counts as a source name too. + e = newTxEnv(t, autoTx("my-plaud"), srv.URL) + e.src.items[0].Connector = "my-plaud" + if n, _ := e.svc.TranscribeSweep(ctx, e.opts()); n != 1 { + t.Fatalf("connector-name opt-in queued %d", n) + } +} + +func TestTranscribeSweepRespectsBudgetWithoutRecordingSkips(t *testing.T) { + srv := audioServer(t, http.StatusOK, txAudio) + ctx := context.Background() + e := newTxEnv(t, autoTx("plaud"), srv.URL) + e.fake.Result.Duration = time.Minute + e.src.items = []Item{ + {Source: "plaud", ID: "a", Pending: true, DurationMS: 60000, RecordedAt: e.now.Add(-time.Minute)}, + {Source: "plaud", ID: "a", Pending: true, DurationMS: 60000, RecordedAt: e.now.Add(-time.Minute)}, + {Source: "plaud", ID: "b", Pending: true, DurationMS: 60000, RecordedAt: e.now.Add(-2 * time.Minute)}, + {Source: "plaud", ID: "c", Pending: true, DurationMS: 60000, RecordedAt: e.now.Add(-3 * time.Minute)}, + } + tx := autoTx("plaud") + tx.DailyMaxItems = 2 + e.svc.UpdateConfig(config.InboxConfig{Transcription: tx}) + if n, _ := e.svc.TranscribeSweep(ctx, e.opts()); n != 2 || e.fake.Calls() != 2 { + t.Fatalf("daily cap: queued %d calls %d (duplicate listing must be deduped)", n, e.fake.Calls()) + } + if _, ok := e.store.Get("plaud", "c"); ok { + t.Fatal("the sweep must leave capped items untouched, not record a skip") + } + // Monthly minutes: 2 of 3 used and each item needs 1, so one more fits. + tx.DailyMaxItems = 0 + tx.MonthlyBudgetMinutes = 3 + e.svc.UpdateConfig(config.InboxConfig{Transcription: tx}) + e.now = e.now.Add(24 * time.Hour) + e.src.items = append(e.src.items, Item{Source: "plaud", ID: "d", Pending: true, DurationMS: 60000}) + if n, _ := e.svc.TranscribeSweep(ctx, e.opts()); n != 1 { + t.Fatalf("monthly budget: queued %d", n) + } + if n, _ := e.svc.TranscribeSweep(ctx, e.opts()); n != 0 { + t.Fatalf("over budget: queued %d", n) + } +} + +func TestTranscribeSweepBackoffAndMaxAttempts(t *testing.T) { + srv := audioServer(t, http.StatusOK, txAudio) + ctx := context.Background() + e := newTxEnv(t, autoTx("plaud"), srv.URL) + boom := errors.New("bad audio") + e.fake.Errs = []error{boom, boom, boom, boom} + if n, _ := e.svc.TranscribeSweep(ctx, e.opts()); n != 1 { + t.Fatalf("first run queued %d", n) + } + if n, _ := e.svc.TranscribeSweep(ctx, e.opts()); n != 0 { + t.Fatal("a failed item waits for its backoff") + } + e.now = e.now.Add(autoRetryBase) + if n, _ := e.svc.TranscribeSweep(ctx, e.opts()); n != 1 { + t.Fatal("retry after one hour") + } + e.now = e.now.Add(autoRetryBase) + if n, _ := e.svc.TranscribeSweep(ctx, e.opts()); n != 0 { + t.Fatal("second retry waits two hours") + } + e.now = e.now.Add(autoRetryBase) + if n, _ := e.svc.TranscribeSweep(ctx, e.opts()); n != 1 { + t.Fatal("second retry after two hours") + } + e.now = e.now.Add(24 * time.Hour) + if n, _ := e.svc.TranscribeSweep(ctx, e.opts()); n != 0 { + t.Fatalf("max %d attempts", TranscriptionMaxAttempts) + } + if rec, _ := e.store.Get("plaud", "rec-1"); rec.Attempts != TranscriptionMaxAttempts || rec.Status != TranscriptionFailed { + t.Fatalf("record = %+v", rec) + } + // The owner can still retry by hand. + e.fake.Errs = nil + if _, err := e.svc.Transcribe(ctx, e.item(t), e.opts()); err != nil { + t.Fatal(err) + } + if rec, _ := e.store.Get("plaud", "rec-1"); rec.Status != TranscriptionDone { + t.Fatalf("manual retry = %+v", rec) + } +} + +func TestAutoDue(t *testing.T) { + now := time.Date(2026, 11, 15, 12, 0, 0, 0, time.UTC) + cases := []struct { + name string + st *TranscriptionState + want bool + }{ + {"never ran", nil, true}, + {"empty", &TranscriptionState{}, true}, + {"done", &TranscriptionState{Status: TranscriptionDone, Attempts: 1}, false}, + {"running", &TranscriptionState{Status: TranscriptionRunning, Attempts: 1}, false}, + {"budget same day", &TranscriptionState{Status: TranscriptionSkipped, Reason: SkipBudget + ": x", Attempts: 1, UpdatedAt: now.Add(-time.Hour)}, false}, + {"budget next day", &TranscriptionState{Status: TranscriptionSkipped, Reason: SkipBudget + ": x", Attempts: 1, UpdatedAt: now.Add(-24 * time.Hour)}, true}, + {"too long", &TranscriptionState{Status: TranscriptionSkipped, Reason: SkipTooLong + ": x", Attempts: 1, UpdatedAt: now.Add(-48 * time.Hour)}, false}, + {"failed early", &TranscriptionState{Status: TranscriptionFailed, Attempts: 1, UpdatedAt: now.Add(-time.Minute)}, false}, + {"failed due", &TranscriptionState{Status: TranscriptionFailed, Attempts: 1, UpdatedAt: now.Add(-time.Hour)}, true}, + {"attempts used", &TranscriptionState{Status: TranscriptionFailed, Attempts: TranscriptionMaxAttempts, UpdatedAt: now.Add(-48 * time.Hour)}, false}, + } + for _, tc := range cases { + if got := autoDue(tc.st, now); got != tc.want { + t.Errorf("%s: autoDue = %v, want %v", tc.name, got, tc.want) + } + } +} + +func TestTranscriptionEgressHosts(t *testing.T) { + if got := TranscriptionEgressHosts(config.InboxTranscription{}); len(got) != 0 || got == nil { + t.Fatalf("disabled egress = %#v", got) + } + if got := TranscriptionEgressHosts(activeTx()); len(got) != 1 || got[0] != transcribe.DeepgramHost { + t.Fatalf("enabled egress = %v", got) + } + e := newTxEnv(t, activeTx(), "http://audio.invalid") + if st := e.svc.TranscriptionSettings(); len(st.EgressHosts) != 1 || st.EgressHosts[0] != "api.deepgram.com" { + t.Fatalf("settings egress = %v", st.EgressHosts) + } +} + +func TestTranscribeAuditDetail(t *testing.T) { + got := TranscribeAudit{Source: "plaud", ItemID: "r", Provider: "deepgram", Model: "nova-3", AudioSeconds: 61, Outcome: "done"}.Detail() + want := "source=plaud, item=r, provider=deepgram, model=nova-3, audio_seconds=61, estimated_usd=0.0000, outcome=done" + if got != want { + t.Fatalf("detail = %q, want %q", got, want) + } +} diff --git a/src/pkg/meetings/quorumproc/quorumproc.go b/src/pkg/meetings/quorumproc/quorumproc.go new file mode 100644 index 0000000000..6427a0222b --- /dev/null +++ b/src/pkg/meetings/quorumproc/quorumproc.go @@ -0,0 +1,511 @@ +// Package quorumproc supervises the Quorum child process (hivecommons/hive#18289, +// hivecommons/quorum docs/contract.md, quorum.contract/v1). +// +// Quorum runs inside the hive container as a child of the hive process, the +// same way hive runs the spektacular binary. It listens on loopback only and +// requires a bearer token that hive generates once per boot and hands to the +// child through QUORUM_TOKEN_FILE. Secrets Quorum needs (the Vexa tenant key, +// the optional Discord bot token) are written in plaintext to owner-only +// files in a private runtime directory that is removed on Stop; their +// durable, keystore-sealed copies stay under /data/secrets. +// +// The supervisor starts the child, polls GET /healthz, restarts the child +// with exponential backoff when it exits or stops answering, and stops it on +// shutdown. +package quorumproc + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "log/slog" + "net/http" + "os" + "os/exec" + "path/filepath" + "strings" + "sync" + "syscall" + "time" +) + +const ( + // DefaultNode is the interpreter for a .mjs/.js Quorum bundle. + DefaultNode = "node" + // DefaultHealthInterval is how often the supervisor polls /healthz. + DefaultHealthInterval = 10 * time.Second + // DefaultHealthTimeout bounds one /healthz request. + DefaultHealthTimeout = 3 * time.Second + // DefaultUnhealthyThreshold is how many consecutive failed health checks + // of a child that was healthy make the supervisor restart it. + DefaultUnhealthyThreshold = 3 + // DefaultStartupTimeout is how long a new child may take to answer + // /healthz before it is restarted. + DefaultStartupTimeout = 60 * time.Second + // DefaultBackoffMin is the first restart delay. + DefaultBackoffMin = time.Second + // DefaultBackoffMax caps the restart delay. + DefaultBackoffMax = time.Minute + // DefaultStableAfter is how long a child must stay healthy before the + // restart backoff resets. + DefaultStableAfter = 2 * time.Minute + // DefaultStopGrace is how long a child gets after SIGTERM before it is + // killed. + DefaultStopGrace = 10 * time.Second + + tokenBytes = 32 + secretFileMode = 0o600 + tokenFileName = "token" + vexaKeyFileName = "vexa_api_key" + discordFileName = "discord_token" +) + +// passthroughEnv are the parent environment variables the child inherits. +// Everything else (GitHub tokens, hive secrets) is withheld; Quorum gets +// only what the contract names plus what node needs to run and reach the +// network through the pod's proxy. +var passthroughEnv = []string{ + "PATH", "HOME", "TZ", "LANG", "LC_ALL", "TMPDIR", + "HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "http_proxy", "https_proxy", "no_proxy", + "NODE_EXTRA_CA_CERTS", "SSL_CERT_FILE", "SSL_CERT_DIR", +} + +// Options configures a Supervisor. +type Options struct { + // Binary is the Quorum bundle (.mjs/.js, run through Node) or an + // executable. Required. + Binary string + // Node is the interpreter for a bundle. Empty means DefaultNode. + Node string + // Listen is the loopback host:port (QUORUM_LISTEN). Required. + Listen string + // VexaURL is QUORUM_VEXA_URL. Empty leaves it unset. + VexaURL string + // VexaAPIKey is the plaintext tenant key written to + // QUORUM_VEXA_API_KEY_FILE. Empty leaves it unset. + VexaAPIKey string + // DiscordToken is written to QUORUM_DISCORD_TOKEN_FILE. Empty leaves + // it unset (Discord tools answer platform_unavailable). + DiscordToken string + // TranscribeURL is QUORUM_TRANSCRIBE_URL. Empty leaves it unset. + TranscribeURL string + // RuntimeDir is the parent of the private runtime directory. Empty means + // os.TempDir(). + RuntimeDir string + // Logger receives lifecycle events and the child's stderr. Nil discards. + Logger *slog.Logger + + HealthInterval time.Duration + HealthTimeout time.Duration + UnhealthyThreshold int + StartupTimeout time.Duration + BackoffMin time.Duration + BackoffMax time.Duration + StableAfter time.Duration + StopGrace time.Duration + + // Command overrides the argv used to start the child (tests). + Command []string + // Getenv overrides os.Getenv for the passthrough environment (tests). + Getenv func(string) string +} + +// Supervisor owns one Quorum child. +type Supervisor struct { + opts Options + argv []string + token string + dir string + env []string + client *http.Client + logger *slog.Logger + + mu sync.Mutex + healthy bool + restarts int + lastErr string + started bool + cancel context.CancelFunc + done chan struct{} + healthCh chan struct{} +} + +// Status is a snapshot of the supervisor for the dashboard and logs. +type Status struct { + Healthy bool `json:"healthy"` + Restarts int `json:"restarts"` + LastErr string `json:"last_error,omitempty"` + BaseURL string `json:"base_url"` +} + +func durOr(v, def time.Duration) time.Duration { + if v > 0 { + return v + } + return def +} + +// New validates opts, generates the per-boot bearer token and writes the +// runtime files. Nothing is started until Start. +func New(opts Options) (*Supervisor, error) { + if strings.TrimSpace(opts.Listen) == "" { + return nil, errors.New("quorumproc: listen address is required") + } + argv := opts.Command + if len(argv) == 0 { + if strings.TrimSpace(opts.Binary) == "" { + return nil, errors.New("quorumproc: quorum binary is required") + } + argv = []string{opts.Binary, "serve"} + if ext := strings.ToLower(filepath.Ext(opts.Binary)); ext == ".mjs" || ext == ".js" { + node := opts.Node + if node == "" { + node = DefaultNode + } + argv = []string{node, opts.Binary, "serve"} + } + } + opts.HealthInterval = durOr(opts.HealthInterval, DefaultHealthInterval) + opts.HealthTimeout = durOr(opts.HealthTimeout, DefaultHealthTimeout) + opts.StartupTimeout = durOr(opts.StartupTimeout, DefaultStartupTimeout) + opts.BackoffMin = durOr(opts.BackoffMin, DefaultBackoffMin) + opts.BackoffMax = durOr(opts.BackoffMax, DefaultBackoffMax) + opts.StableAfter = durOr(opts.StableAfter, DefaultStableAfter) + opts.StopGrace = durOr(opts.StopGrace, DefaultStopGrace) + if opts.UnhealthyThreshold <= 0 { + opts.UnhealthyThreshold = DefaultUnhealthyThreshold + } + if opts.Getenv == nil { + opts.Getenv = os.Getenv + } + logger := opts.Logger + if logger == nil { + logger = slog.New(slog.DiscardHandler) + } + + token, err := newToken() + if err != nil { + return nil, err + } + dir, err := os.MkdirTemp(opts.RuntimeDir, "hive-quorum-") + if err != nil { + return nil, fmt.Errorf("quorumproc: creating runtime dir: %w", err) + } + s := &Supervisor{ + opts: opts, + argv: argv, + token: token, + dir: dir, + client: &http.Client{Timeout: opts.HealthTimeout}, + logger: logger, + healthCh: make(chan struct{}), + } + if err := s.writeRuntimeFiles(); err != nil { + _ = os.RemoveAll(dir) + return nil, err + } + return s, nil +} + +func newToken() (string, error) { + b := make([]byte, tokenBytes) + if _, err := rand.Read(b); err != nil { + return "", fmt.Errorf("quorumproc: generating token: %w", err) + } + return hex.EncodeToString(b), nil +} + +// writeRuntimeFiles writes the token and secrets and builds the child env. +func (s *Supervisor) writeRuntimeFiles() error { + var env []string + for _, k := range passthroughEnv { + if v := s.opts.Getenv(k); v != "" { + env = append(env, k+"="+v) + } + } + // QUORUM_* tunables an operator set on hive (the contract's numeric + // limits) pass through; the ones below are hive-owned and win. + for _, kv := range os.Environ() { + if strings.HasPrefix(kv, "QUORUM_") { + env = append(env, kv) + } + } + write := func(name, value string) (string, error) { + path := filepath.Join(s.dir, name) + if err := os.WriteFile(path, []byte(value), secretFileMode); err != nil { + return "", fmt.Errorf("quorumproc: writing %s: %w", name, err) + } + return path, nil + } + tokenPath, err := write(tokenFileName, s.token) + if err != nil { + return err + } + env = append(env, "QUORUM_LISTEN="+s.opts.Listen, "QUORUM_TOKEN_FILE="+tokenPath) + if s.opts.VexaURL != "" { + env = append(env, "QUORUM_VEXA_URL="+s.opts.VexaURL) + } + if s.opts.VexaAPIKey != "" { + p, err := write(vexaKeyFileName, s.opts.VexaAPIKey) + if err != nil { + return err + } + env = append(env, "QUORUM_VEXA_API_KEY_FILE="+p) + } + if s.opts.DiscordToken != "" { + p, err := write(discordFileName, s.opts.DiscordToken) + if err != nil { + return err + } + env = append(env, "QUORUM_DISCORD_TOKEN_FILE="+p) + } + if s.opts.TranscribeURL != "" { + env = append(env, "QUORUM_TRANSCRIBE_URL="+s.opts.TranscribeURL) + } + s.env = env + return nil +} + +// BaseURL is the Quorum base URL (http://). +func (s *Supervisor) BaseURL() string { return "http://" + s.opts.Listen } + +// Token is the per-boot bearer token. A secret; never log it. +func (s *Supervisor) Token() string { return s.token } + +// TokenFile is the path of the 0600 token file. +func (s *Supervisor) TokenFile() string { return filepath.Join(s.dir, tokenFileName) } + +// Env returns a copy of the child's environment (tests and diagnostics; it +// holds file paths, never secret values). +func (s *Supervisor) Env() []string { return append([]string(nil), s.env...) } + +// Status returns a snapshot. +func (s *Supervisor) Status() Status { + s.mu.Lock() + defer s.mu.Unlock() + return Status{Healthy: s.healthy, Restarts: s.restarts, LastErr: s.lastErr, BaseURL: s.BaseURL()} +} + +// Healthy reports whether the last health check passed. +func (s *Supervisor) Healthy() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.healthy +} + +// WaitHealthy blocks until the child first answers /healthz or ctx ends. +func (s *Supervisor) WaitHealthy(ctx context.Context) error { + select { + case <-s.healthCh: + return nil + case <-ctx.Done(): + return ctx.Err() + } +} + +// Start launches the supervision loop. It returns at once; the loop runs +// until ctx ends or Stop is called. Start may be called once. +func (s *Supervisor) Start(ctx context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + if s.started { + return errors.New("quorumproc: already started") + } + s.started = true + ctx, s.cancel = context.WithCancel(ctx) + s.done = make(chan struct{}) + go s.loop(ctx) + return nil +} + +// Stop stops the child and removes the runtime directory. Safe to call more +// than once and before Start. +func (s *Supervisor) Stop() { + s.mu.Lock() + cancel, done := s.cancel, s.done + s.mu.Unlock() + if cancel != nil { + cancel() + <-done + } + _ = os.RemoveAll(s.dir) +} + +func (s *Supervisor) setHealthy(ok bool) { + s.mu.Lock() + defer s.mu.Unlock() + if ok && !s.healthy { + select { + case <-s.healthCh: + default: + close(s.healthCh) + } + } + s.healthy = ok +} + +func (s *Supervisor) setErr(err error) { + s.mu.Lock() + defer s.mu.Unlock() + s.lastErr = err.Error() +} + +func (s *Supervisor) loop(ctx context.Context) { + defer close(s.done) + backoff := s.opts.BackoffMin + for { + healthySince, err := s.runOnce(ctx) + s.setHealthy(false) + if ctx.Err() != nil { + return + } + if err != nil { + s.setErr(err) + } + if !healthySince.IsZero() && time.Since(healthySince) >= s.opts.StableAfter { + backoff = s.opts.BackoffMin + } + s.mu.Lock() + s.restarts++ + s.mu.Unlock() + s.logger.Warn("[meetings] quorum child stopped; restarting", "error", err, "backoff", backoff.String()) + select { + case <-ctx.Done(): + return + case <-time.After(backoff): + } + backoff *= 2 + if backoff > s.opts.BackoffMax { + backoff = s.opts.BackoffMax + } + } +} + +// runOnce starts one child and supervises it until it exits, stops answering +// health checks, or ctx ends. It returns when the child became healthy (zero +// if never) and why it stopped. +func (s *Supervisor) runOnce(ctx context.Context) (time.Time, error) { + cmd := exec.Command(s.argv[0], s.argv[1:]...) + cmd.Env = s.env + cmd.Dir = s.dir + cmd.Stdout = &logWriter{logger: s.logger, stream: "stdout"} + cmd.Stderr = &logWriter{logger: s.logger, stream: "stderr"} + if err := cmd.Start(); err != nil { + return time.Time{}, fmt.Errorf("starting quorum: %w", err) + } + s.logger.Info("[meetings] quorum child started", "pid", cmd.Process.Pid, "listen", s.opts.Listen) + exited := make(chan error, 1) + go func() { exited <- cmd.Wait() }() + + stop := func() { + _ = cmd.Process.Signal(syscall.SIGTERM) + select { + case <-exited: + case <-time.After(s.opts.StopGrace): + _ = cmd.Process.Kill() + <-exited + } + } + + var healthySince time.Time + failures := 0 + startDeadline := time.Now().Add(s.opts.StartupTimeout) + tick := time.NewTicker(s.healthPollInterval()) + defer tick.Stop() + for { + select { + case <-ctx.Done(): + stop() + return healthySince, nil + case err := <-exited: + return healthySince, fmt.Errorf("quorum exited: %v", err) + case <-tick.C: + err := s.check(ctx) + if err == nil { + failures = 0 + if healthySince.IsZero() { + healthySince = time.Now() + s.logger.Info("[meetings] quorum healthy", "listen", s.opts.Listen) + tick.Reset(s.opts.HealthInterval) + } + s.setHealthy(true) + continue + } + if healthySince.IsZero() { + if time.Now().After(startDeadline) { + stop() + return time.Time{}, fmt.Errorf("quorum not healthy within %s: %w", s.opts.StartupTimeout, err) + } + continue + } + s.setHealthy(false) + failures++ + if failures >= s.opts.UnhealthyThreshold { + stop() + return healthySince, fmt.Errorf("quorum failed %d health checks: %w", failures, err) + } + } + } +} + +// healthPollInterval polls faster while waiting for the first healthy +// answer, so a fresh child is usable quickly. +func (s *Supervisor) healthPollInterval() time.Duration { + const startupPollDivisor = 10 + return max(s.opts.HealthInterval/startupPollDivisor, time.Millisecond) +} + +func (s *Supervisor) check(ctx context.Context) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, s.BaseURL()+"/healthz", nil) + if err != nil { + return err + } + req.Header.Set("Authorization", "Bearer "+s.token) + resp, err := s.client.Do(req) + if err != nil { + return err + } + _ = resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("healthz answered %d", resp.StatusCode) + } + return nil +} + +// logWriter forwards child output to the logger line by line. +type logWriter struct { + logger *slog.Logger + stream string + mu sync.Mutex + buf []byte +} + +// maxLogLine bounds one buffered child output line. +const maxLogLine = 8 << 10 + +func (w *logWriter) Write(p []byte) (int, error) { + w.mu.Lock() + defer w.mu.Unlock() + w.buf = append(w.buf, p...) + for { + i := bytes.IndexByte(w.buf, '\n') + if i < 0 { + if len(w.buf) > maxLogLine { + w.emit(w.buf) + w.buf = w.buf[:0] + } + return len(p), nil + } + w.emit(w.buf[:i]) + w.buf = w.buf[i+1:] + } +} + +func (w *logWriter) emit(line []byte) { + if s := strings.TrimSpace(string(line)); s != "" { + w.logger.Info("[meetings] quorum", "stream", w.stream, "line", s) + } +} diff --git a/src/pkg/meetings/quorumproc/quorumproc_test.go b/src/pkg/meetings/quorumproc/quorumproc_test.go new file mode 100644 index 0000000000..c61dc57d55 --- /dev/null +++ b/src/pkg/meetings/quorumproc/quorumproc_test.go @@ -0,0 +1,426 @@ +package quorumproc + +import ( + "bytes" + "context" + "errors" + "fmt" + "log/slog" + "net" + "net/http" + "os" + "os/signal" + "path/filepath" + "strings" + "sync" + "syscall" + "testing" + "time" + + "github.com/hivecommons/hive/internal/testutil" +) + +// helperEnv selects the fake Quorum behaviour when the test binary is +// re-executed as the child. It is a QUORUM_* name so the supervisor's +// passthrough hands it to the child. +const helperEnv = "QUORUM_TEST_HELPER" + +func TestMain(m *testing.M) { + if mode := os.Getenv(helperEnv); mode != "" { + os.Exit(runHelper(mode)) + } + os.Exit(m.Run()) +} + +// runHelper is the fake `quorum serve`: it honours QUORUM_LISTEN and +// QUORUM_TOKEN_FILE the way the contract says the real one does. +func runHelper(mode string) int { + if mode == "exit" { + fmt.Fprintln(os.Stderr, "fake quorum: exiting") + return 1 + } + if mode == "ignore-term" { + signal.Ignore(syscall.SIGTERM) + } + tok, err := os.ReadFile(os.Getenv("QUORUM_TOKEN_FILE")) + if err != nil { + return 2 + } + want := "Bearer " + string(tok) + mux := http.NewServeMux() + mux.HandleFunc("/healthz", func(w http.ResponseWriter, r *http.Request) { + if mode == "unhealthy" || mode == "ignore-term" { + w.WriteHeader(http.StatusServiceUnavailable) + return + } + if r.Header.Get("Authorization") != want { + w.WriteHeader(http.StatusUnauthorized) + return + } + fmt.Fprintln(os.Stdout, "healthz ok") + _, _ = w.Write([]byte(`{"ok":true}`)) + }) + srv := &http.Server{Addr: os.Getenv("QUORUM_LISTEN"), Handler: mux, ReadHeaderTimeout: time.Second} + if mode == "ignore-term" { + // Never exit on SIGTERM, so the supervisor must kill after grace. + _ = srv.ListenAndServe() + return 0 + } + go func() { + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGTERM) + <-sig + _ = srv.Close() + }() + if err := srv.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { + return 3 + } + return 0 +} + +func freeAddr(t *testing.T) string { + t.Helper() + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + addr := l.Addr().String() + _ = l.Close() + return addr +} + +type syncBuf struct { + mu sync.Mutex + b bytes.Buffer +} + +func (s *syncBuf) Write(p []byte) (int, error) { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.Write(p) +} + +func (s *syncBuf) String() string { + s.mu.Lock() + defer s.mu.Unlock() + return s.b.String() +} + +func fastOpts(t *testing.T, mode string) (Options, *syncBuf) { + t.Helper() + t.Setenv(helperEnv, mode) + logs := &syncBuf{} + return Options{ + Listen: freeAddr(t), + Command: []string{os.Args[0]}, + RuntimeDir: t.TempDir(), + Logger: slog.New(slog.NewTextHandler(logs, nil)), + HealthInterval: 50 * time.Millisecond, + HealthTimeout: time.Second, + StartupTimeout: 5 * time.Second, + BackoffMin: 10 * time.Millisecond, + BackoffMax: 40 * time.Millisecond, + StableAfter: time.Hour, + StopGrace: time.Second, + UnhealthyThreshold: 2, + }, logs +} + +func waitFor(t *testing.T, what string, cond func() bool) { + t.Helper() + testutil.Eventually(t, 10*time.Second, cond, "timed out waiting for %s", what) +} + +func TestNewValidates(t *testing.T) { + if _, err := New(Options{}); err == nil { + t.Fatal("want an error without a listen address") + } + if _, err := New(Options{Listen: "127.0.0.1:1"}); err == nil { + t.Fatal("want an error without a binary") + } + if _, err := New(Options{Listen: "127.0.0.1:1", Binary: "/q", RuntimeDir: filepath.Join(t.TempDir(), "missing")}); err == nil { + t.Fatal("want an error when the runtime dir cannot be created") + } +} + +func TestNewArgvAndDefaults(t *testing.T) { + dir := t.TempDir() + s, err := New(Options{Listen: "127.0.0.1:1", Binary: "/opt/quorum.mjs", RuntimeDir: dir}) + if err != nil { + t.Fatal(err) + } + defer s.Stop() + if got := strings.Join(s.argv, " "); got != "node /opt/quorum.mjs serve" { + t.Fatalf("argv = %q", got) + } + if s.opts.HealthInterval != DefaultHealthInterval || s.opts.UnhealthyThreshold != DefaultUnhealthyThreshold || + s.opts.BackoffMax != DefaultBackoffMax || s.opts.StopGrace != DefaultStopGrace { + t.Fatalf("defaults not applied: %+v", s.opts) + } + s2, err := New(Options{Listen: "127.0.0.1:1", Binary: "/opt/q.js", Node: "/usr/bin/node22", RuntimeDir: dir}) + if err != nil { + t.Fatal(err) + } + defer s2.Stop() + if got := strings.Join(s2.argv, " "); got != "/usr/bin/node22 /opt/q.js serve" { + t.Fatalf("argv = %q", got) + } + s3, err := New(Options{Listen: "127.0.0.1:1", Binary: "/usr/local/bin/quorum", RuntimeDir: dir}) + if err != nil { + t.Fatal(err) + } + defer s3.Stop() + if got := strings.Join(s3.argv, " "); got != "/usr/local/bin/quorum serve" { + t.Fatalf("argv = %q", got) + } +} + +func TestRuntimeFilesAndEnv(t *testing.T) { + t.Setenv("QUORUM_CHAT_MAX_CHARS", "1500") + env := map[string]string{"PATH": "/bin", "HIVE_GITHUB_TOKEN": "ghp_secret", "HTTPS_PROXY": "http://proxy"} + s, err := New(Options{ + Listen: "127.0.0.1:8765", + Binary: "/q.mjs", + VexaURL: "https://vexa", + VexaAPIKey: "tenant", + DiscordToken: "discord", + TranscribeURL: "http://whisper", + RuntimeDir: t.TempDir(), + Getenv: func(k string) string { return env[k] }, + }) + if err != nil { + t.Fatal(err) + } + if len(s.Token()) != 2*tokenBytes { + t.Fatalf("token length %d", len(s.Token())) + } + got := map[string]string{} + for _, kv := range s.Env() { + k, v, _ := strings.Cut(kv, "=") + got[k] = v + } + if _, leaked := got["HIVE_GITHUB_TOKEN"]; leaked { + t.Fatal("hive secrets must not reach the child") + } + for k, want := range map[string]string{ + "PATH": "/bin", "HTTPS_PROXY": "http://proxy", "QUORUM_LISTEN": "127.0.0.1:8765", + "QUORUM_VEXA_URL": "https://vexa", "QUORUM_TRANSCRIBE_URL": "http://whisper", "QUORUM_CHAT_MAX_CHARS": "1500", + } { + if got[k] != want { + t.Errorf("%s = %q, want %q", k, got[k], want) + } + } + for envKey, want := range map[string]string{"QUORUM_TOKEN_FILE": s.Token(), "QUORUM_VEXA_API_KEY_FILE": "tenant", "QUORUM_DISCORD_TOKEN_FILE": "discord"} { + path := got[envKey] + fi, err := os.Stat(path) + if err != nil || fi.Mode().Perm() != secretFileMode { + t.Fatalf("%s at %s: %v %v", envKey, path, fi, err) + } + b, _ := os.ReadFile(path) + if string(b) != want { + t.Errorf("%s holds %q", envKey, b) + } + } + if s.TokenFile() != got["QUORUM_TOKEN_FILE"] { + t.Fatal("TokenFile mismatch") + } + if fi, _ := os.Stat(s.dir); fi.Mode().Perm() != 0o700 { + t.Fatalf("runtime dir mode %v", fi.Mode().Perm()) + } + if s.BaseURL() != "http://127.0.0.1:8765" { + t.Fatal(s.BaseURL()) + } + s.Stop() + if _, err := os.Stat(s.dir); !os.IsNotExist(err) { + t.Fatal("Stop must remove the runtime dir") + } +} + +func TestOptionalEnvOmitted(t *testing.T) { + s, err := New(Options{Listen: "127.0.0.1:1", Binary: "/q", RuntimeDir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + defer s.Stop() + for _, kv := range s.Env() { + for _, k := range []string{"QUORUM_VEXA_URL=", "QUORUM_VEXA_API_KEY_FILE=", "QUORUM_DISCORD_TOKEN_FILE=", "QUORUM_TRANSCRIBE_URL="} { + if strings.HasPrefix(kv, k) { + t.Errorf("unset option leaked as %s", kv) + } + } + } +} + +func TestSuperviseHealthyThenStop(t *testing.T) { + opts, logs := fastOpts(t, "serve") + s, err := New(opts) + if err != nil { + t.Fatal(err) + } + if err := s.Start(context.Background()); err != nil { + t.Fatal(err) + } + if err := s.Start(context.Background()); err == nil { + t.Fatal("a second Start must fail") + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := s.WaitHealthy(ctx); err != nil { + t.Fatalf("WaitHealthy: %v (logs: %s)", err, logs.String()) + } + if !s.Healthy() || !s.Status().Healthy { + t.Fatal("want healthy") + } + waitFor(t, "child stdout in logs", func() bool { return strings.Contains(logs.String(), "healthz ok") }) + s.Stop() + s.Stop() + if s.Healthy() { + t.Fatal("stopped supervisor must not report healthy") + } + if st := s.Status(); st.Restarts != 0 { + t.Fatalf("restarts = %d", st.Restarts) + } +} + +func TestSuperviseRestartsAfterExit(t *testing.T) { + opts, logs := fastOpts(t, "exit") + s, err := New(opts) + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancel(context.Background()) + if err := s.Start(ctx); err != nil { + t.Fatal(err) + } + waitFor(t, "restarts", func() bool { return s.Status().Restarts >= 3 }) + if st := s.Status(); !strings.Contains(st.LastErr, "exited") { + t.Fatalf("last error %q", st.LastErr) + } + waitFor(t, "stderr in logs", func() bool { return strings.Contains(logs.String(), "fake quorum: exiting") }) + cancel() + s.Stop() + short, cancel2 := context.WithTimeout(context.Background(), 20*time.Millisecond) + defer cancel2() + if err := s.WaitHealthy(short); err == nil { + t.Fatal("never-healthy child must time out WaitHealthy") + } +} + +func TestSuperviseStartFailure(t *testing.T) { + opts, _ := fastOpts(t, "serve") + opts.Command = []string{filepath.Join(t.TempDir(), "no-such-binary")} + s, err := New(opts) + if err != nil { + t.Fatal(err) + } + if err := s.Start(context.Background()); err != nil { + t.Fatal(err) + } + waitFor(t, "start error", func() bool { return strings.Contains(s.Status().LastErr, "starting quorum") }) + s.Stop() +} + +func TestSuperviseStartupTimeout(t *testing.T) { + opts, _ := fastOpts(t, "unhealthy") + opts.StartupTimeout = 200 * time.Millisecond + s, err := New(opts) + if err != nil { + t.Fatal(err) + } + if err := s.Start(context.Background()); err != nil { + t.Fatal(err) + } + waitFor(t, "startup timeout", func() bool { return strings.Contains(s.Status().LastErr, "not healthy within") }) + s.Stop() +} + +func TestSuperviseKillsChildIgnoringSIGTERM(t *testing.T) { + opts, _ := fastOpts(t, "ignore-term") + opts.StartupTimeout = 200 * time.Millisecond + opts.StopGrace = 100 * time.Millisecond + s, err := New(opts) + if err != nil { + t.Fatal(err) + } + if err := s.Start(context.Background()); err != nil { + t.Fatal(err) + } + waitFor(t, "kill after grace", func() bool { return s.Status().Restarts >= 1 }) + s.Stop() +} + +// TestSuperviseRestartsUnhealthyChild drives the "was healthy, then stopped +// answering" path against an in-process health server whose answer flips. +func TestSuperviseRestartsUnhealthyChild(t *testing.T) { + var mu sync.Mutex + status := http.StatusOK + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + srv := &http.Server{ReadHeaderTimeout: time.Second, Handler: http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + defer mu.Unlock() + w.WriteHeader(status) + })} + go func() { _ = srv.Serve(l) }() + defer srv.Close() + + opts, _ := fastOpts(t, "") + opts.Listen = l.Addr().String() + opts.Command = []string{"sleep", "30"} + opts.StableAfter = time.Millisecond + s, err := New(opts) + if err != nil { + t.Fatal(err) + } + if err := s.Start(context.Background()); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + if err := s.WaitHealthy(ctx); err != nil { + t.Fatal(err) + } + mu.Lock() + status = http.StatusInternalServerError + mu.Unlock() + waitFor(t, "unhealthy restart", func() bool { + return s.Status().Restarts >= 1 && strings.Contains(s.Status().LastErr, "health checks") + }) + mu.Lock() + status = http.StatusOK + mu.Unlock() + waitFor(t, "healthy again", s.Healthy) + s.Stop() +} + +func TestCheckBadURL(t *testing.T) { + s := &Supervisor{opts: Options{Listen: "bad host:1"}, client: http.DefaultClient} + if err := s.check(context.Background()); err == nil { + t.Fatal("want an error for an unparseable URL") + } +} + +func TestLogWriterSplitsAndCaps(t *testing.T) { + logs := &syncBuf{} + w := &logWriter{logger: slog.New(slog.NewTextHandler(logs, nil)), stream: "stderr"} + _, _ = w.Write([]byte("one\ntw")) + _, _ = w.Write([]byte("o\n\n")) + _, _ = w.Write(bytes.Repeat([]byte("x"), maxLogLine+1)) + out := logs.String() + if !strings.Contains(out, "line=one") || !strings.Contains(out, "line=two") || !strings.Contains(out, "xxxx") { + t.Fatalf("log output %q", out) + } + if len(w.buf) != 0 { + t.Fatal("oversized partial line must be flushed") + } +} + +func TestStopBeforeStart(t *testing.T) { + s, err := New(Options{Listen: "127.0.0.1:1", Binary: "/q", RuntimeDir: t.TempDir()}) + if err != nil { + t.Fatal(err) + } + s.Stop() +} diff --git a/src/pkg/research/pool.go b/src/pkg/research/pool.go new file mode 100644 index 0000000000..43ef2951e2 --- /dev/null +++ b/src/pkg/research/pool.go @@ -0,0 +1,359 @@ +// Package research is the research agent pool the meetings feature kicks +// (#18288, parent #18284; Quorum contract section 3). +// +// A research request from a meeting becomes a queued kick to the least busy +// of N read-only research agents (research-1..N). The agent reads the web, +// prints "[FINDING] " per result and "[COMPLETE] " when +// done; those output signals come back through the runtime's kick observers +// tagged with the kick ID, and Pool.Observe turns them into Events. +// +// The agents are ADVISORY with no repo checkouts: the hive proxy refuses +// every GitHub write (anything but GET/HEAD/OPTIONS and git fetch) for an +// ADVISORY agent and its token tier is read-only, so "no repo writes, no +// GitHub mutations" holds even if the prompt is ignored. +package research + +import ( + "context" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + "strconv" + "strings" + "sync" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/config" +) + +const ( + // Role is the research agents' role. + Role = "research" + // NamePrefix is the research agents' name prefix: research-1..N. + NamePrefix = "research-" + // DefaultPoolSize is the pool size when meetings.research_pool_size is + // unset or not positive. + DefaultPoolSize = 2 + // MaxPoolSize caps meetings.research_pool_size. + MaxPoolSize = 8 + // Mode is the research agents' GitHub interaction tier: read-only. + Mode = "ADVISORY" + + // maxQueryLen bounds a research query typed into an agent, in bytes. + maxQueryLen = 2000 + // maxTrackedKicks bounds the kicks Observe can attribute at once; the + // oldest is forgotten first. + maxTrackedKicks = 256 + // kickIDPrefix and kickIDRandomBytes shape the pool's kick IDs: + // "rsch-" + 16 hex characters. + kickIDPrefix = "rsch-" + kickIDRandomBytes = 8 + + displayName = "research" + emoji = "🔎" + description = "Read-only research agent for live meetings. Reads public web sources and reports findings; never writes to a repository or GitHub." +) + +// Event kinds Observe delivers. They map onto the contract's research_update +// statuses: queued, running, done, failed; finding carries one interim +// result. +const ( + EventQueued = "queued" + EventRunning = "running" + EventFinding = "finding" + EventDone = "done" + EventFailed = "failed" +) + +// ErrUnsupportedRuntime means the runtime does not offer queued kicks and +// fan-out observers (the sidecar runtime today). +var ErrUnsupportedRuntime = errors.New("research pool: runtime does not support queued kicks") + +// ErrEmptyQuery means Dispatch was given no query. +var ErrEmptyQuery = errors.New("research pool: empty query") + +// ErrNoAgents means every research agent is unavailable. +var ErrNoAgents = errors.New("research pool: no research agent available") + +// Runtime is what the pool needs from the agent runtime. *agent.Manager +// satisfies it; NewPool type-asserts a crux.Runtime to it. +type Runtime interface { + crux.KickQueuer + crux.KickObserverRegistry + AddAgent(name string, cfg config.AgentConfig) + GetStatusFast(name string) (*crux.Process, error) + Start(ctx context.Context, name string) error +} + +// Event is one research progress update for a kick the pool dispatched. +type Event struct { + // KickID is what Dispatch returned. + KickID string + // Agent is the research agent running it. + Agent string + // Kind is one of the Event* constants. + Kind string + // Text is the finding line (EventFinding), the summary (EventDone) or + // the reason (EventFailed). It may be empty for EventDone when the + // agent's turn ended without a [COMPLETE] line. + Text string + // Meta is the map passed to Dispatch. + Meta map[string]string +} + +// PoolSize resolves meetings.research_pool_size: DefaultPoolSize when not +// positive, MaxPoolSize at most. +func PoolSize(configured int) int { + switch { + case configured <= 0: + return DefaultPoolSize + case configured > MaxPoolSize: + return MaxPoolSize + default: + return configured + } +} + +// AgentNames returns research-1..n. +func AgentNames(n int) []string { + names := make([]string, 0, n) + for i := 1; i <= n; i++ { + names = append(names, NamePrefix+strconv.Itoa(i)) + } + return names +} + +// IsResearchAgent reports whether name is a research pool agent. +func IsResearchAgent(name string) bool { + n, err := strconv.Atoi(strings.TrimPrefix(name, NamePrefix)) + return strings.HasPrefix(name, NamePrefix) && err == nil && n > 0 +} + +// AgentConfig is the research agent definition. base supplies the backend +// and model (and anything else the caller wants kept); the read-only fields +// are always forced, so a base cannot widen the agent's access. +func AgentConfig(base config.AgentConfig) config.AgentConfig { + cfg := base + noRepos := false + cfg.Role = Role + cfg.Mode = Mode + cfg.IncludeRepos = &noRepos + cfg.OnDemand = true + cfg.Converse = nil + if cfg.DisplayName == "" { + cfg.DisplayName = displayName + } + if cfg.Emoji == "" { + cfg.Emoji = emoji + } + if cfg.Description == "" { + cfg.Description = description + } + return cfg +} + +// Prompt is the kick a research agent receives for one query. The markers +// are spelled out in words rather than written literally: the runtime only +// treats a marker that starts a line as a signal, and a pane that wraps an +// echoed prompt could otherwise start a line with one. +func Prompt(query string) string { + q := strings.TrimSpace(query) + if len(q) > maxQueryLen { + q = strings.ToValidUTF8(q[:maxQueryLen], "") + } + return strings.Join([]string{ + "You are a research assistant for a live meeting. Research the question below using public web sources only.", + "Rules: read web pages and documentation; do not clone, edit or push to any repository; do not create, comment on, label or close GitHub issues or pull requests; do not run anything that changes a remote system.", + "Report as you go. For each useful result print one line that starts with the word FINDING in square brackets, then one sentence and its source URL.", + "When you are done print one line that starts with the word COMPLETE in square brackets, then a summary of at most three sentences.", + "If you cannot answer, print one line that starts with the word ERROR in square brackets, then the reason.", + "Keep it short: the meeting is waiting.", + "", + "Question: " + q, + }, "\n") +} + +// Pool dispatches research queries to the research agents. +type Pool struct { + rt Runtime + names []string + + // randRead fills kick ID randomness; crypto/rand.Read unless a test + // swaps it. + randRead func([]byte) (int, error) + + mu sync.Mutex + tracked map[string]*trackedKick + order []string +} + +type trackedKick struct { + agent string + meta map[string]string + done bool +} + +// NewPool wraps rt (normally the hive's crux.Runtime) for a pool of size +// agents. It returns ErrUnsupportedRuntime when rt lacks queued kicks. +func NewPool(rt any, size int) (*Pool, error) { + r, ok := rt.(Runtime) + if !ok { + return nil, ErrUnsupportedRuntime + } + return &Pool{rt: r, names: AgentNames(PoolSize(size)), randRead: rand.Read, tracked: make(map[string]*trackedKick)}, nil +} + +// Agents returns the pool's agent names. +func (p *Pool) Agents() []string { return append([]string(nil), p.names...) } + +// Ensure registers the research agents with the runtime (AddAgent is a no-op +// for one that already exists). Call it only when meetings are enabled: the +// pool's agents exist for nothing else. +func (p *Pool) Ensure(base config.AgentConfig) { + cfg := AgentConfig(base) + for _, name := range p.names { + p.rt.AddAgent(name, cfg) + } +} + +// Dispatch queues query on the least busy research agent and returns the +// kick ID. A stopped agent is started first. meta rides along on every Event +// for this kick (meeting ID, research ID, ...). +func (p *Pool) Dispatch(ctx context.Context, query string, meta map[string]string) (string, error) { + if strings.TrimSpace(query) == "" { + return "", ErrEmptyQuery + } + agent, err := p.pick() + if err != nil { + return "", err + } + if st, err := p.rt.GetStatusFast(agent); err == nil && st != nil && st.State != crux.StateRunning { + if err := p.rt.Start(ctx, agent); err != nil { + return "", fmt.Errorf("research pool: start %s: %w", agent, err) + } + } + // Track under our own ID BEFORE enqueueing: the runtime emits + // kick-queued from inside EnqueueKick, and an untracked ID would drop it. + id, err := p.newID() + if err != nil { + return "", err + } + p.track(id, agent, meta) + if _, err := p.rt.EnqueueKick(agent, id, Prompt(query)); err != nil { + p.untrack(id) + return "", fmt.Errorf("research pool: queue on %s: %w", agent, err) + } + return id, nil +} + +// newID returns a fresh kick ID for a research kick. +func (p *Pool) newID() (string, error) { + b := make([]byte, kickIDRandomBytes) + if _, err := p.randRead(b); err != nil { + return "", fmt.Errorf("research pool: kick id: %w", err) + } + return kickIDPrefix + hex.EncodeToString(b), nil +} + +func (p *Pool) untrack(id string) { + p.mu.Lock() + defer p.mu.Unlock() + delete(p.tracked, id) + for i, v := range p.order { + if v == id { + p.order = append(p.order[:i], p.order[i+1:]...) + break + } + } +} + +// pick returns the available agent with the fewest waiting plus running +// kicks; ties go to the lowest-numbered agent. +func (p *Pool) pick() (string, error) { + best, bestLoad := "", 0 + for _, name := range p.names { + st, err := p.rt.GetStatusFast(name) + if err != nil || st == nil || st.Paused || st.State == crux.StatePaused { + continue + } + load := p.rt.KickQueueDepth(name) + if p.rt.InFlightKickID(name) != "" { + load++ + } + if best == "" || load < bestLoad { + best, bestLoad = name, load + } + } + if best == "" { + return "", ErrNoAgents + } + return best, nil +} + +func (p *Pool) track(id, agent string, meta map[string]string) { + cp := make(map[string]string, len(meta)) + for k, v := range meta { + cp[k] = v + } + p.mu.Lock() + defer p.mu.Unlock() + p.tracked[id] = &trackedKick{agent: agent, meta: cp} + p.order = append(p.order, id) + for len(p.order) > maxTrackedKicks { + delete(p.tracked, p.order[0]) + p.order = p.order[1:] + } +} + +// Observe registers fn for progress on every kick this pool dispatched and +// returns its remover. Each kick yields queued, running, any number of +// finding, then exactly one of done or failed. +func (p *Pool) Observe(fn func(Event)) (remove func()) { + return p.rt.AddKickObserver(func(ev crux.KickEvent) { + if out, ok := p.translate(ev); ok { + fn(out) + } + }) +} + +// translate maps a runtime kick event for a tracked kick to a pool Event. +func (p *Pool) translate(ev crux.KickEvent) (Event, bool) { + if ev.KickID == "" { + return Event{}, false + } + p.mu.Lock() + defer p.mu.Unlock() + tk, ok := p.tracked[ev.KickID] + if !ok || tk.done { + return Event{}, false + } + out := Event{KickID: ev.KickID, Agent: tk.agent, Meta: tk.meta} + switch ev.Event { + case crux.KickEventQueued: + out.Kind = EventQueued + case crux.KickEventDelivered: + out.Kind = EventRunning + case crux.KickEventOutputSignal: + switch ev.Signal { + case crux.OutputSignalFinding: + out.Kind, out.Text = EventFinding, ev.Text + case crux.OutputSignalComplete: + out.Kind, out.Text = EventDone, ev.Text + case crux.OutputSignalError: + out.Kind, out.Text = EventFailed, ev.Text + default: + return Event{}, false + } + case crux.KickEventCompleted: + out.Kind = EventDone + case crux.KickEventFailed: + out.Kind, out.Text = EventFailed, ev.Detail + default: + return Event{}, false + } + if out.Kind == EventDone || out.Kind == EventFailed { + tk.done = true + } + return out, true +} diff --git a/src/pkg/research/pool_test.go b/src/pkg/research/pool_test.go new file mode 100644 index 0000000000..775a44dde6 --- /dev/null +++ b/src/pkg/research/pool_test.go @@ -0,0 +1,291 @@ +package research + +import ( + "context" + "errors" + "strings" + "testing" + + "github.com/hivecommons/hive/crux" + "github.com/hivecommons/hive/pkg/config" +) + +type fakeRuntime struct { + added map[string]config.AgentConfig + states map[string]*crux.Process + depth map[string]int + inflight map[string]string + started []string + startErr error + queueErr error + queued []string // agent:kickID + prompts []string + observer func(crux.KickEvent) +} + +func newFake(names ...string) *fakeRuntime { + f := &fakeRuntime{ + added: map[string]config.AgentConfig{}, + states: map[string]*crux.Process{}, + depth: map[string]int{}, + inflight: map[string]string{}, + } + for _, n := range names { + f.states[n] = &crux.Process{Name: n, State: crux.StateRunning} + } + return f +} + +func (f *fakeRuntime) EnqueueKick(agent, kickID, message string) (string, error) { + if f.queueErr != nil { + return "", f.queueErr + } + f.queued = append(f.queued, agent+":"+kickID) + f.prompts = append(f.prompts, message) + f.depth[agent]++ + if f.observer != nil { + f.observer(crux.KickEvent{Agent: agent, Event: crux.KickEventQueued, KickID: kickID}) + } + return kickID, nil +} +func (f *fakeRuntime) KickStatus(string) (crux.KickStatus, bool) { return crux.KickStatus{}, false } +func (f *fakeRuntime) KickQueueDepth(agent string) int { return f.depth[agent] } +func (f *fakeRuntime) InFlightKickID(agent string) string { return f.inflight[agent] } +func (f *fakeRuntime) AddKickObserver(fn func(crux.KickEvent)) func() { + f.observer = fn + return func() { f.observer = nil } +} +func (f *fakeRuntime) AddAgent(name string, cfg config.AgentConfig) { f.added[name] = cfg } +func (f *fakeRuntime) GetStatusFast(name string) (*crux.Process, error) { + st, ok := f.states[name] + if !ok { + return nil, errors.New("agent not found") + } + return st, nil +} +func (f *fakeRuntime) Start(_ context.Context, name string) error { + if f.startErr != nil { + return f.startErr + } + f.started = append(f.started, name) + f.states[name].State = crux.StateRunning + return nil +} + +func TestPoolSizeAndNames(t *testing.T) { + for in, want := range map[int]int{-1: DefaultPoolSize, 0: DefaultPoolSize, 3: 3, MaxPoolSize + 5: MaxPoolSize} { + if got := PoolSize(in); got != want { + t.Errorf("PoolSize(%d) = %d, want %d", in, got, want) + } + } + if got := strings.Join(AgentNames(3), ","); got != "research-1,research-2,research-3" { + t.Errorf("AgentNames(3) = %s", got) + } + for name, want := range map[string]bool{"research-1": true, "research-12": true, "research-0": false, "research-x": false, "research": false, "scanner": false} { + if got := IsResearchAgent(name); got != want { + t.Errorf("IsResearchAgent(%q) = %v, want %v", name, got, want) + } + } +} + +func TestAgentConfigForcesReadOnly(t *testing.T) { + yes := true + cfg := AgentConfig(config.AgentConfig{Backend: "claude", Model: "m", Mode: "ISSUES_AND_PRS", IncludeRepos: &yes, Converse: &yes, Role: "scanner"}) + if cfg.Mode != Mode || cfg.Role != Role || cfg.IncludeRepos == nil || *cfg.IncludeRepos || !cfg.OnDemand || cfg.Converse != nil { + t.Fatalf("read-only fields not forced: %+v", cfg) + } + if cfg.Backend != "claude" || cfg.Model != "m" { + t.Fatalf("base backend/model not kept: %+v", cfg) + } + if cfg.DisplayName == "" || cfg.Emoji == "" || cfg.Description == "" { + t.Fatalf("defaults not applied: %+v", cfg) + } + kept := AgentConfig(config.AgentConfig{DisplayName: "d", Emoji: "e", Description: "x"}) + if kept.DisplayName != "d" || kept.Emoji != "e" || kept.Description != "x" { + t.Fatalf("base labels overwritten: %+v", kept) + } + if m, ok := crux.ParseAgentMode(Mode); !ok || m != crux.ModeAdvisory { + t.Fatalf("Mode %q is not ADVISORY", Mode) + } +} + +func TestPromptNeverStartsALineWithAMarker(t *testing.T) { + p := Prompt(" what changed in vLLM 0.9? ") + if !strings.HasSuffix(p, "Question: what changed in vLLM 0.9?") { + t.Fatalf("prompt does not end with the query: %q", p) + } + if strings.Contains(p, "[FINDING]") || strings.Contains(p, "[COMPLETE]") || strings.Contains(p, "[ERROR]") { + t.Fatal("prompt spells a literal marker; a wrapped echo could be read as a signal") + } + long := Prompt(strings.Repeat("é", maxQueryLen)) + q := long[strings.LastIndex(long, "Question: ")+len("Question: "):] + if len(q) > maxQueryLen || !strings.HasPrefix(q, "é") || strings.ContainsRune(q, '�') { + t.Fatalf("long query not truncated cleanly: %d bytes", len(q)) + } +} + +func TestNewPoolRejectsUnsupportedRuntime(t *testing.T) { + if _, err := NewPool(struct{}{}, 2); !errors.Is(err, ErrUnsupportedRuntime) { + t.Fatalf("err = %v, want ErrUnsupportedRuntime", err) + } +} + +func TestEnsureAddsEveryAgent(t *testing.T) { + f := newFake() + p, err := NewPool(f, 0) + if err != nil { + t.Fatal(err) + } + p.Ensure(config.AgentConfig{Backend: "claude"}) + if len(f.added) != DefaultPoolSize || f.added["research-2"].Mode != Mode { + t.Fatalf("added = %+v", f.added) + } + if got := strings.Join(p.Agents(), ","); got != "research-1,research-2" { + t.Fatalf("Agents() = %s", got) + } +} + +func TestDispatchPicksLeastBusyAndStartsStopped(t *testing.T) { + f := newFake("research-1", "research-2", "research-3") + f.depth["research-1"] = 1 + f.inflight["research-2"] = "busy" + f.states["research-3"].State = crux.StateStopped + p, _ := NewPool(f, 3) + + id, err := p.Dispatch(context.Background(), "q1", map[string]string{"meeting_id": "m1"}) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(id, kickIDPrefix) || f.queued[0] != "research-3:"+id { + t.Fatalf("queued = %v id = %s, want research-3 (idle)", f.queued, id) + } + if len(f.started) != 1 || f.started[0] != "research-3" { + t.Fatalf("started = %v, want the stopped agent started", f.started) + } + if !strings.Contains(f.prompts[0], "Question: q1") { + t.Fatalf("prompt = %q", f.prompts[0]) + } + + // Ties go to the lowest-numbered agent; a paused agent is skipped. + f.states["research-1"].Paused = true + f.depth["research-3"] = 1 // now research-2 (1 in flight) ties research-3 (1 queued) + if _, err := p.Dispatch(context.Background(), "q2", nil); err != nil { + t.Fatal(err) + } + if got := f.queued[1]; !strings.HasPrefix(got, "research-2:") { + t.Fatalf("second dispatch went to %s, want research-2", got) + } +} + +func TestDispatchErrors(t *testing.T) { + f := newFake("research-1") + p, _ := NewPool(f, 1) + ctx := context.Background() + if _, err := p.Dispatch(ctx, " ", nil); !errors.Is(err, ErrEmptyQuery) { + t.Errorf("empty query err = %v", err) + } + + f.queueErr = crux.ErrKickQueueFull + if _, err := p.Dispatch(ctx, "q", nil); !errors.Is(err, crux.ErrKickQueueFull) { + t.Errorf("queue err = %v", err) + } + if len(p.tracked) != 0 || len(p.order) != 0 { + t.Errorf("a failed enqueue stayed tracked: %v", p.order) + } + f.queueErr = nil + + f.states["research-1"].State = crux.StateStopped + f.startErr = errors.New("no tmux") + if _, err := p.Dispatch(ctx, "q", nil); err == nil || !strings.Contains(err.Error(), "no tmux") { + t.Errorf("start err = %v", err) + } + f.startErr = nil + + p.randRead = func([]byte) (int, error) { return 0, errors.New("entropy gone") } + if _, err := p.Dispatch(ctx, "q", nil); err == nil || !strings.Contains(err.Error(), "entropy gone") { + t.Errorf("id err = %v", err) + } + + empty, _ := NewPool(newFake(), 1) + if _, err := empty.Dispatch(ctx, "q", nil); !errors.Is(err, ErrNoAgents) { + t.Errorf("no agents err = %v", err) + } +} + +func TestObserveTranslatesLifecycle(t *testing.T) { + f := newFake("research-1") + p, _ := NewPool(f, 1) + var got []Event + remove := p.Observe(func(ev Event) { got = append(got, ev) }) + + id, err := p.Dispatch(context.Background(), "q", map[string]string{"rid": "r1"}) + if err != nil { + t.Fatal(err) + } + emit := func(ev crux.KickEvent) { + ev.Agent = "research-1" + f.observer(ev) + } + emit(crux.KickEvent{Event: crux.KickEventDelivered, KickID: id}) + emit(crux.KickEvent{Event: crux.KickEventOutputSignal, KickID: id, Signal: crux.OutputSignalStatus, Text: "reading"}) + emit(crux.KickEvent{Event: crux.KickEventOutputSignal, KickID: id, Signal: crux.OutputSignalFinding, Text: "f1"}) + emit(crux.KickEvent{Event: crux.KickEventArchived, KickID: id}) + emit(crux.KickEvent{Event: crux.KickEventOutputSignal, KickID: "other", Signal: crux.OutputSignalFinding}) + emit(crux.KickEvent{Event: crux.KickEventDelivered}) + emit(crux.KickEvent{Event: crux.KickEventOutputSignal, KickID: id, Signal: crux.OutputSignalComplete, Text: "summary"}) + emit(crux.KickEvent{Event: crux.KickEventCompleted, KickID: id}) // after done: ignored + + kinds := make([]string, 0, len(got)) + for _, ev := range got { + kinds = append(kinds, ev.Kind) + if ev.KickID != id || ev.Agent != "research-1" || ev.Meta["rid"] != "r1" { + t.Fatalf("event = %+v", ev) + } + } + if strings.Join(kinds, ",") != "queued,running,finding,done" { + t.Fatalf("kinds = %v", kinds) + } + if got[2].Text != "f1" || got[3].Text != "summary" { + t.Fatalf("texts = %q %q", got[2].Text, got[3].Text) + } + + // Turn end without [COMPLETE] is done; [ERROR] and kick-failed are failed. + got = nil + a, _ := p.Dispatch(context.Background(), "a", nil) + b, _ := p.Dispatch(context.Background(), "b", nil) + c, _ := p.Dispatch(context.Background(), "c", nil) + emit(crux.KickEvent{Event: crux.KickEventCompleted, KickID: a, Detail: "ended"}) + emit(crux.KickEvent{Event: crux.KickEventOutputSignal, KickID: b, Signal: crux.OutputSignalError, Text: "blocked"}) + emit(crux.KickEvent{Event: crux.KickEventFailed, KickID: c, Detail: "timeout"}) + want := map[string]string{a: EventDone + ":", b: EventFailed + ":blocked", c: EventFailed + ":timeout"} + for _, ev := range got { + if ev.Kind == EventQueued { + continue + } + if w := want[ev.KickID]; w != ev.Kind+":"+ev.Text { + t.Errorf("kick %s: got %s:%s, want %s", ev.KickID, ev.Kind, ev.Text, w) + } + delete(want, ev.KickID) + } + if len(want) != 0 { + t.Errorf("missing terminal events for %v", want) + } + + remove() + if f.observer != nil { + t.Fatal("remover did not unregister") + } +} + +func TestTrackBoundsMemory(t *testing.T) { + p, _ := NewPool(newFake("research-1"), 1) + for i := 0; i < maxTrackedKicks+3; i++ { + p.track(strings.Repeat("x", i+1), "research-1", nil) + } + if len(p.tracked) != maxTrackedKicks || len(p.order) != maxTrackedKicks { + t.Fatalf("tracked %d / order %d, want %d", len(p.tracked), len(p.order), maxTrackedKicks) + } + if _, ok := p.tracked["x"]; ok { + t.Fatal("oldest tracked kick not evicted") + } +}